feat(manager/regex)!: allow arbitrary regex groups for templates

lib/manager/regex/__fixtures__/ansible.yml

@@ -1,5 +1,7 @@
 prometheus_image: "prom/prometheus"  // depName gets initially set
+
+prometheus_registry: "docker.io"  // depName gets initially set
+prometheus_repository: "prom/prometheus"  // depName gets initially set
 prometheus_version: "v2.21.0" // currentValue get set
 
-someother_image: "" // will not be set as group value is null/empty string
 someother_version: "0.12.0" // overwrites currentValue as later values take precedence.

lib/manager/regex/__snapshots__/index.spec.ts.snap

@@ -257,6 +257,28 @@ Object {
 }
 `;
 
+exports[`manager/regex/index extracts with combination strategy and non standard capture groups 1`] = `
+Object {
+  "datasourceTemplate": "docker",
+  "depNameTemplate": "{{{ registry }}}/{{{ repository }}}",
+  "deps": Array [
+    Object {
+      "currentValue": "v2.21.0",
+      "datasource": "docker",
+      "depName": "docker.io/prom/prometheus",
+      "replaceString": "prometheus_version: \\"v2.21.0\\" //",
+    },
+  ],
+  "matchStrings": Array [
+    "prometheus_registry:\\\\s*\\"(?<registry>.*)\\"\\\\s*\\\\/\\\\/",
+    "prometheus_repository:\\\\s*\\"(?<repository>.*)\\"\\\\s*\\\\/\\\\/",
+    "prometheus_tag:\\\\s*\\"(?<tag>.*)\\"\\\\s*\\\\/\\\\/",
+    "prometheus_version:\\\\s*\\"(?<currentValue>.*)\\"\\\\s*\\\\/\\\\/",
+  ],
+  "matchStringsStrategy": "combination",
+}
+`;
+
 exports[`manager/regex/index extracts with combination strategy and registry url 1`] = `
 Object {
   "datasourceTemplate": "helm",

lib/manager/regex/index.spec.ts

@@ -206,6 +206,29 @@ describe('manager/regex/index', () => {
     expect(res).toMatchSnapshot();
     expect(res.deps).toHaveLength(1);
   });
+
+  it('extracts with combination strategy and non standard capture groups', async () => {
+    const config: CustomExtractConfig = {
+      matchStrings: [
+        'prometheus_registry:\\s*"(?<registry>.*)"\\s*\\/\\/',
+        'prometheus_repository:\\s*"(?<repository>.*)"\\s*\\/\\/',
+        'prometheus_tag:\\s*"(?<tag>.*)"\\s*\\/\\/',
+        'prometheus_version:\\s*"(?<currentValue>.*)"\\s*\\/\\/',
+      ],
+      matchStringsStrategy: 'combination',
+      datasourceTemplate: 'docker',
+      depNameTemplate: '{{{ registry }}}/{{{ repository }}}',
+    };
+    const res = await extractPackageFile(
+      ansibleYamlContent,
+      'ansible.yml',
+      config
+    );
+    expect(res.deps).toHaveLength(1);
+    expect(res.deps[0].depName).toEqual('docker.io/prom/prometheus');
+    expect(res).toMatchSnapshot();
+  });
+
   it('extracts with combination strategy and multiple matches', async () => {
     const config: CustomExtractConfig = {
       matchStrings: [

lib/manager/regex/index.ts

@@ -25,7 +25,10 @@ const validMatchFields = [
   'depType',
 ];
 
-const mergeFields = ['registryUrls', ...validMatchFields];
+interface ExtractionTemplate {
+  groups: Record<string, string>;
+  replaceString: string;
+}
 
 function regexMatchAll(regex: RegExp, content: string): RegExpMatchArray[] {
   const matches: RegExpMatchArray[] = [];
@@ -40,13 +43,12 @@ function regexMatchAll(regex: RegExp, content: string): RegExpMatchArray[] {
 }
 
 function createDependency(
-  matchResult: RegExpMatchArray,
-  combinedGroups: Record<string, string>,
+  extractionTemplate: ExtractionTemplate,
   config: CustomExtractConfig,
   dep?: PackageDependency
 ): PackageDependency {
   const dependency = dep || {};
-  const { groups } = matchResult;
+  const { groups, replaceString } = extractionTemplate;
 
   function updateDependency(field: string, value: string): void {
     switch (field) {
@@ -69,11 +71,7 @@ function createDependency(
     const fieldTemplate = `${field}Template`;
     if (config[fieldTemplate]) {
       try {
-        const compiled = template.compile(
-          config[fieldTemplate],
-          combinedGroups ?? groups,
-          false
-        );
+        const compiled = template.compile(config[fieldTemplate], groups, false);
         updateDependency(field, compiled);
       } catch (err) {
         logger.warn(
@@ -86,26 +84,10 @@ function createDependency(
       updateDependency(field, groups[field]);
     }
   }
-  dependency.replaceString = String(matchResult[0]);
+  dependency.replaceString = replaceString;
   return dependency;
 }
 
-function mergeDependency(deps: PackageDependency[]): PackageDependency {
-  const result: PackageDependency = {};
-  deps.forEach((dep) => {
-    mergeFields.forEach((field) => {
-      if (dep[field]) {
-        result[field] = dep[field];
-        // save the line replaceString of the section which contains the current Value for a speed up lookup during the replace phase
-        if (field === 'currentValue') {
-          result.replaceString = dep.replaceString;
-        }
-      }
-    });
-  });
-  return result;
-}
-
 function handleAny(
   content: string,
   packageFile: string,
@@ -114,25 +96,29 @@ function handleAny(
   return config.matchStrings
     .map((matchString) => regEx(matchString, 'g'))
     .flatMap((regex) => regexMatchAll(regex, content)) // match all regex to content, get all matches, reduce to single array
-    .map((matchResult) => createDependency(matchResult, null, config));
+    .map((matchResult) =>
+      createDependency(
+        { groups: matchResult.groups, replaceString: matchResult[0] },
+        config
+      )
+    );
 }
 
 function mergeGroups(
   mergedGroup: Record<string, string>,
   secondGroup: Record<string, string>
 ): Record<string, string> {
-  const resultGroup = Object.create(null); // prevent prototype pollution
+  return { ...mergedGroup, ...secondGroup };
+}
 
-  Object.keys(mergedGroup).forEach(
-    // eslint-disable-next-line no-return-assign
-    (key) => (resultGroup[key] = mergedGroup[key])
-  );
-  Object.keys(secondGroup).forEach((key) => {
-    if (secondGroup[key] && secondGroup[key] !== '') {
-      resultGroup[key] = secondGroup[key];
-    }
-  });
-  return resultGroup;
+export function mergeExtractionTemplate(
+  base: ExtractionTemplate,
+  addition: ExtractionTemplate
+): ExtractionTemplate {
+  return {
+    groups: mergeGroups(base.groups, addition.groups),
+    replaceString: addition.replaceString ?? base.replaceString,
+  };
 }
 
 function handleCombination(
@@ -148,20 +134,13 @@ function handleCombination(
     return [];
   }
 
-  const combinedGroup = matches
-    .map((match) => match.groups)
-    .reduce((mergedGroup, currentGroup) =>
-      mergeGroups(mergedGroup, currentGroup)
-    );
-
-  // TODO: this seems to be buggy behavior, needs to be checked #11387
-  const dep = matches
-    .map((match) => createDependency(match, combinedGroup, config))
-    .reduce(
-      (mergedDep, currentDep) => mergeDependency([mergedDep, currentDep]),
-      {}
-    ); // merge fields of dependencies
-  return [dep];
+  const extraction = matches
+    .map((match) => ({
+      groups: match.groups,
+      replaceString: match?.groups?.currentValue ? match[0] : undefined,
+    }))
+    .reduce((base, addition) => mergeExtractionTemplate(base, addition));
+  return [createDependency(extraction, config)];
 }
 
 function handleRecursive(
@@ -182,8 +161,10 @@ function handleRecursive(
     // if we have a depName and a currentValue with have the minimal viable definition
     if (match?.groups?.depName && match?.groups?.currentValue) {
       return createDependency(
-        match,
-        mergeGroups(combinedGroups, match.groups),
+        {
+          groups: mergeGroups(combinedGroups, match.groups),
+          replaceString: match[0],
+        },
         config
       );
     }