feat(manager/gradle): add support for apply from statements (#16030)

* add synchronous local file read

* add tests

* add support for apply from statements

* add safeguard to prevent breakout from localDir

* add safeguard to allow "apply from" only with official Gradle file types

* fix: use async io

* fix test coverage

* Update lib/modules/manager/gradle/extract.spec.ts

Co-authored-by: Sergei Zharinov <zharinov@users.noreply.github.com>

* Update lib/modules/manager/gradle/extract.spec.ts

Co-authored-by: Sergei Zharinov <zharinov@users.noreply.github.com>

* Update lib/modules/manager/gradle/extract.spec.ts

Co-authored-by: Sergei Zharinov <zharinov@users.noreply.github.com>

* re-add try/catch

* add istanbul ignore

* Update lib/modules/manager/gradle/parser.ts

Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>

* fs tests: remove obsolete block

* remove istanbul ignore next

* Update lib/modules/manager/gradle/extract.ts

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>

* tests: replace mockImplementationOnce() with mockRejectedValueOnce()

* scriptFile: explicitly set null for undefined values

Co-authored-by: Michael Kriese <michael.kriese@visualon.de>
Co-authored-by: Rhys Arkins <rhys@arkins.net>
Co-authored-by: Sergei Zharinov <zharinov@users.noreply.github.com>
Co-authored-by: HonkingGoose <34918129+HonkingGoose@users.noreply.github.com>

lib/modules/manager/gradle/extract.spec.ts

@@ -1,17 +1,24 @@
 import { Fixtures } from '../../../../test/fixtures';
-import { fs } from '../../../../test/util';
+import { fs, logger } from '../../../../test/util';
 import type { ExtractConfig } from '../types';
+import * as parser from './parser';
 import { extractAllPackageFiles } from '.';
 
 jest.mock('../../../util/fs');
 
 function mockFs(files: Record<string, string>): void {
   fs.readLocalFile.mockImplementation((fileName: string): Promise<string> => {
     const content = files?.[fileName];
-    return typeof content === 'string'
-      ? Promise.resolve(content)
-      : Promise.reject(`File not found: ${fileName}`);
+    return Promise.resolve(content ?? '');
   });
+
+  fs.getSiblingFileName.mockImplementation(
+    (existingFileNameWithPath: string, otherFileName: string) => {
+      return existingFileNameWithPath
+        .slice(0, existingFileNameWithPath.lastIndexOf('/') + 1)
+        .concat(otherFileName);
+    }
+  );
 }
 
 describe('modules/manager/gradle/extract', () => {
@@ -33,6 +40,19 @@ describe('modules/manager/gradle/extract', () => {
     expect(res).toBeNull();
   });
 
+  it('logs a warning in case parseGradle throws an exception', async () => {
+    const filename = 'build.gradle';
+    const err = new Error('unknown');
+
+    jest.spyOn(parser, 'parseGradle').mockRejectedValueOnce(err);
+    await extractAllPackageFiles({} as ExtractConfig, [filename]);
+
+    expect(logger.logger.warn).toHaveBeenCalledWith(
+      { err, config: {}, packageFile: filename },
+      `Failed to process Gradle file: ${filename}`
+    );
+  });
+
   it('extracts from cross-referenced files', async () => {
     mockFs({
       'gradle.properties': 'baz=1.2.3',
@@ -605,4 +625,191 @@ describe('modules/manager/gradle/extract', () => {
       },
     ]);
   });
+
+  it('loads further scripts using apply from statements', async () => {
+    const buildFile = `
+      buildscript {
+          repositories {
+              mavenCentral()
+          }
+
+          apply from: "\${someDir}/libs1.gradle"
+          apply from: file("gradle/libs2.gradle")
+          apply from: "gradle/libs3.gradle"
+          apply from: file("gradle/non-existing.gradle")
+
+          dependencies {
+              classpath "com.google.protobuf:protobuf-java:\${protoBufVersion}"
+              classpath "com.google.guava:guava:\${guavaVersion}"
+              classpath "io.jsonwebtoken:jjwt-api:0.11.2"
+
+              classpath "org.junit.jupiter:junit-jupiter-api:\${junitVersion}"
+              classpath "org.junit.jupiter:junit-jupiter-engine:\${junitVersion}"
+          }
+      }
+    `;
+
+    mockFs({
+      'gradleX/libs1.gradle': "ext.junitVersion = '5.5.2'",
+      'gradle/libs2.gradle': "ext.protoBufVersion = '3.18.2'",
+      'gradle/libs3.gradle': "ext.guavaVersion = '30.1-jre'",
+      'build.gradle': buildFile,
+      'gradle.properties': 'someDir=gradleX',
+    });
+
+    const res = await extractAllPackageFiles({} as ExtractConfig, [
+      'gradleX/libs1.gradle',
+      'gradle/libs2.gradle',
+      // 'gradle/libs3.gradle', is intentionally not listed here
+      'build.gradle',
+      'gradle.properties',
+    ]);
+
+    expect(res).toMatchObject([
+      { packageFile: 'gradle.properties' },
+      {
+        packageFile: 'build.gradle',
+        deps: [{ depName: 'io.jsonwebtoken:jjwt-api' }],
+      },
+      {
+        packageFile: 'gradle/libs2.gradle',
+        deps: [
+          {
+            depName: 'com.google.protobuf:protobuf-java',
+            currentValue: '3.18.2',
+            managerData: { packageFile: 'gradle/libs2.gradle' },
+          },
+        ],
+      },
+      {
+        packageFile: 'gradleX/libs1.gradle',
+        deps: [
+          {
+            depName: 'org.junit.jupiter:junit-jupiter-api',
+            currentValue: '5.5.2',
+            managerData: { packageFile: 'gradleX/libs1.gradle' },
+          },
+          {
+            depName: 'org.junit.jupiter:junit-jupiter-engine',
+            currentValue: '5.5.2',
+            managerData: { packageFile: 'gradleX/libs1.gradle' },
+          },
+        ],
+      },
+      {
+        packageFile: 'gradle/libs3.gradle',
+        deps: [
+          {
+            depName: 'com.google.guava:guava',
+            currentValue: '30.1-jre',
+            managerData: { packageFile: 'gradle/libs3.gradle' },
+          },
+        ],
+      },
+    ]);
+  });
+
+  it('apply from works with files in sub-directories', async () => {
+    const buildFile = `
+      buildscript {
+          repositories {
+              mavenCentral()
+          }
+
+          apply from: "gradle/libs4.gradle"
+
+          dependencies {
+              classpath "com.google.protobuf:protobuf-java:\${protoBufVersion}"
+          }
+      }
+    `;
+
+    mockFs({
+      'somesubdir/gradle/libs4.gradle': "ext.protoBufVersion = '3.18.2'",
+      'somesubdir/build.gradle': buildFile,
+    });
+
+    const res = await extractAllPackageFiles({} as ExtractConfig, [
+      'somesubdir/gradle/libs4.gradle',
+      'somesubdir/build.gradle',
+    ]);
+
+    expect(res).toMatchObject([
+      { packageFile: 'somesubdir/build.gradle' },
+      {
+        packageFile: 'somesubdir/gradle/libs4.gradle',
+        deps: [{ depName: 'com.google.protobuf:protobuf-java' }],
+      },
+    ]);
+  });
+
+  it('prevents recursive apply from calls', async () => {
+    mockFs({
+      'build.gradle': "apply from: 'test.gradle'",
+      'test.gradle': "apply from: 'build.gradle'",
+    });
+
+    const res = await extractAllPackageFiles({} as ExtractConfig, [
+      'build.gradle',
+      'test.gradle',
+    ]);
+
+    expect(res).toBeNull();
+  });
+
+  it('prevents inclusion of non-Gradle files', async () => {
+    mockFs({
+      'build.gradle': "apply from: '../../test.non-gradle'",
+    });
+
+    const res = await extractAllPackageFiles({} as ExtractConfig, [
+      'build.gradle',
+    ]);
+
+    expect(res).toBeNull();
+  });
+
+  it('filters duplicate dependency findings', async () => {
+    const buildFile = `
+      apply from: 'test.gradle'
+
+      repositories {
+          mavenCentral()
+      }
+
+      dependencies {
+        implementation "io.jsonwebtoken:jjwt-api:$\{jjwtVersion}"
+        runtimeOnly "io.jsonwebtoken:jjwt-impl:$\{jjwtVersion}"
+      }
+    `;
+
+    const testFile = `
+      ext.jjwtVersion = '0.11.2'
+
+      ext {
+          jjwtApi = "io.jsonwebtoken:jjwt-api:$jjwtVersion"
+      }
+    `;
+
+    mockFs({
+      'build.gradle': buildFile,
+      'test.gradle': testFile,
+    });
+
+    const res = await extractAllPackageFiles({} as ExtractConfig, [
+      'build.gradle',
+      'test.gradle',
+    ]);
+
+    expect(res).toMatchObject([
+      {
+        packageFile: 'test.gradle',
+        deps: [
+          { depName: 'io.jsonwebtoken:jjwt-api' },
+          { depName: 'io.jsonwebtoken:jjwt-impl' },
+        ],
+      },
+      { packageFile: 'build.gradle' },
+    ]);
+  });
 });

lib/modules/manager/gradle/extract.ts

@@ -74,7 +74,7 @@ export async function extractAllPackageFiles(
           deps,
           urls,
           vars: gradleVars,
-        } = parseGradle(content, vars, packageFile);
+        } = await parseGradle(content, vars, packageFile);
         urls.forEach((url) => {
           if (!registryUrls.includes(url)) {
             registryUrls.push(url);
@@ -100,18 +100,33 @@ export async function extractAllPackageFiles(
     const key = dep.managerData?.packageFile;
     // istanbul ignore else
     if (key) {
-      const pkgFile: PackageFile = packageFilesByName[key];
-      const { deps } = pkgFile;
-      deps.push({
-        ...dep,
-        registryUrls: [
-          ...new Set([
-            ...defaultRegistryUrls,
-            ...(dep.registryUrls ?? []),
-            ...registryUrls,
-          ]),
-        ],
-      });
+      let pkgFile = packageFilesByName[key];
+      if (!pkgFile) {
+        pkgFile = {
+          packageFile: key,
+          datasource,
+          deps: [],
+        } as PackageFile;
+      }
+
+      dep.registryUrls = [
+        ...new Set([
+          ...defaultRegistryUrls,
+          ...(dep.registryUrls ?? []),
+          ...registryUrls,
+        ]),
+      ];
+
+      const depAlreadyInPkgFile = pkgFile.deps.some(
+        (item) =>
+          item.depName === dep.depName &&
+          item.managerData?.fileReplacePosition ===
+            dep.managerData?.fileReplacePosition
+      );
+      if (!depAlreadyInPkgFile) {
+        pkgFile.deps.push(dep);
+      }
+
       packageFilesByName[key] = pkgFile;
     } else {
       logger.warn({ dep }, `Failed to process Gradle dependency`);