feat(terraform): use HCL parser and introduce class based extractors

lib/modules/manager/terraform/base.ts

@@ -0,0 +1,54 @@
+import is from '@sindresorhus/is';
+import { regEx } from '../../../util/regex';
+import { TerraformProviderDatasource } from '../../datasource/terraform-provider';
+import type { PackageDependency } from '../types';
+import type { ProviderLock } from './lockfile/types';
+import { getLockedVersion, massageProviderLookupName } from './util';
+
+export abstract class DependencyExtractor {
+  abstract extract(hclRoot: any, locks: ProviderLock[]): PackageDependency[];
+}
+
+export abstract class TerraformProviderExtractor extends DependencyExtractor {
+  sourceExtractionRegex = regEx(
+    /^(?:(?<hostname>(?:[a-zA-Z0-9-_]+\.+)+[a-zA-Z0-9-_]+)\/)?(?:(?<namespace>[^/]+)\/)?(?<type>[^/]+)/
+  );
+
+  analyzeTerraformProvider(
+    dep: PackageDependency,
+    locks: ProviderLock[],
+    depType: string
+  ): PackageDependency {
+    dep.depType = depType;
+    dep.depName = dep.managerData?.moduleName;
+    dep.datasource = TerraformProviderDatasource.id;
+
+    if (is.nonEmptyString(dep.managerData?.source)) {
+      // TODO #7154
+      const source = this.sourceExtractionRegex.exec(dep.managerData!.source);
+      if (!source?.groups) {
+        dep.skipReason = 'unsupported-url';
+        return dep;
+      }
+
+      // buildin providers https://github.com/terraform-providers
+      if (source.groups.namespace === 'terraform-providers') {
+        dep.registryUrls = [`https://releases.hashicorp.com`];
+      } else if (source.groups.hostname) {
+        dep.registryUrls = [`https://${source.groups.hostname}`];
+        dep.packageName = `${source.groups.namespace}/${source.groups.type}`;
+      } else {
+        dep.packageName = dep.managerData?.source;
+      }
+    }
+    massageProviderLookupName(dep);
+
+    dep.lockedVersion = getLockedVersion(dep, locks);
+
+    if (!dep.currentValue) {
+      dep.skipReason = 'no-version';
+    }
+
+    return dep;
+  }
+}

lib/modules/manager/terraform/extract.spec.ts

@@ -401,8 +401,8 @@ describe('modules/manager/terraform/extract', () => {
 
     it('extracts docker resources', async () => {
       const res = await extractPackageFile(docker, 'docker.tf', {});
-      expect(res?.deps).toHaveLength(8);
-      expect(res?.deps.filter((dep) => dep.skipReason)).toHaveLength(5);
+      expect(res?.deps).toHaveLength(6);
+      expect(res?.deps.filter((dep) => dep.skipReason)).toHaveLength(3);
       expect(res?.deps).toIncludeAllPartialMembers([
         {
           autoReplaceStringTemplate:
@@ -414,6 +414,7 @@ describe('modules/manager/terraform/extract', () => {
           replaceString: 'nginx:1.7.8',
         },
         {
+          depType: 'docker_image',
           skipReason: 'invalid-dependency-specification',
         },
         {
@@ -434,6 +435,7 @@ describe('modules/manager/terraform/extract', () => {
           replaceString: 'nginx:1.7.8',
         },
         {
+          depType: 'docker_container',
           skipReason: 'invalid-dependency-specification',
         },
         {
@@ -446,12 +448,6 @@ describe('modules/manager/terraform/extract', () => {
           depType: 'docker_service',
           replaceString: 'repo.mycompany.com:8080/foo-service:v1',
         },
-        {
-          skipReason: 'invalid-dependency-specification',
-        },
-        {
-          skipReason: 'invalid-value',
-        },
       ]);
     });
 
@@ -504,7 +500,10 @@ describe('modules/manager/terraform/extract', () => {
           currentValue: '1.21.5',
           depType: 'kubernetes_job',
         },
-        { skipReason: 'invalid-value' },
+        {
+          depType: 'kubernetes_job',
+          skipReason: 'invalid-dependency-specification',
+        },
         {
           depName: 'nginx',
           currentValue: '1.21.6',

lib/modules/manager/terraform/extract.ts

@@ -1,35 +1,12 @@
-import is from '@sindresorhus/is';
 import { logger } from '../../../logger';
-import { newlineRegex, regEx } from '../../../util/regex';
-import type { ExtractConfig, PackageDependency, PackageFile } from '../types';
-import type { ProviderLock } from './lockfile/types';
-import { extractLocks, findLockFile, readLockFile } from './lockfile/util';
-import { analyseTerraformModule, extractTerraformModule } from './modules';
-import {
-  analyzeTerraformProvider,
-  extractTerraformProvider,
-} from './providers';
-import {
-  analyzeTerraformRequiredProvider,
-  extractTerraformRequiredProviders,
-} from './required-providers';
-import {
-  analyseTerraformVersion,
-  extractTerraformRequiredVersion,
-} from './required-version';
-import {
-  analyseTerraformResource,
-  extractTerraformResource,
-} from './resources';
-import type { ExtractionResult, TerraformManagerData } from './types';
+import type { ExtractConfig, PackageFile } from '../types';
+import dependencyExtractors from './extractors';
+import * as hcl from './hcl';
 import {
   checkFileContainsDependency,
-  getTerraformDependencyType,
+  extractLocksForPackageFile,
 } from './util';
 
-const dependencyBlockExtractionRegex = regEx(
-  /^\s*(?<type>[a-z_]+)\s+("(?<packageName>[^"]+)"\s+)?("(?<terraformName>[^"]+)"\s+)?{\s*$/
-);
 const contentCheckList = [
   'module "',
   'provider "',
@@ -55,104 +32,20 @@ export async function extractPackageFile(
     );
     return null;
   }
-  let deps: PackageDependency<TerraformManagerData>[] = [];
-  try {
-    const lines = content.split(newlineRegex);
-    for (let lineNumber = 0; lineNumber < lines.length; lineNumber += 1) {
-      const line = lines[lineNumber];
-      const terraformDependency = dependencyBlockExtractionRegex.exec(line);
-      if (terraformDependency?.groups) {
-        logger.trace(
-          `Matched ${terraformDependency.groups.type} on line ${lineNumber}`
-        );
-        const tfDepType = getTerraformDependencyType(
-          terraformDependency.groups.type
-        );
-        let result: ExtractionResult | null = null;
-        switch (tfDepType) {
-          case 'required_providers': {
-            result = extractTerraformRequiredProviders(lineNumber, lines);
-            break;
-          }
-          case 'provider': {
-            result = extractTerraformProvider(
-              lineNumber,
-              lines,
-              terraformDependency.groups.packageName
-            );
-            break;
-          }
-          case 'module': {
-            result = extractTerraformModule(
-              lineNumber,
-              lines,
-              terraformDependency.groups.packageName
-            );
-            break;
-          }
-          case 'resource': {
-            result = extractTerraformResource(lineNumber, lines);
-            break;
-          }
-          case 'terraform_version': {
-            result = extractTerraformRequiredVersion(lineNumber, lines);
-            break;
-          }
-          /* istanbul ignore next */
-          default:
-            logger.trace(
-              `Could not identify TerraformDependencyType ${terraformDependency.groups.type} on line ${lineNumber}.`
-            );
-            break;
-        }
-        if (result) {
-          lineNumber = result.lineNumber;
-          deps = deps.concat(result.dependencies);
-          result = null;
-        }
-      }
-    }
-  } catch (err) /* istanbul ignore next */ {
-    logger.warn({ err }, 'Error extracting terraform plugins');
-  }
 
-  const locks: ProviderLock[] = [];
-  const lockFilePath = findLockFile(fileName);
-  if (lockFilePath) {
-    const lockFileContent = await readLockFile(lockFilePath);
-    if (lockFileContent) {
-      const extractedLocks = extractLocks(lockFileContent);
-      if (is.nonEmptyArray(extractedLocks)) {
-        locks.push(...extractedLocks);
-      }
-    }
-  }
+  const dependencies = [];
+  const hclMap = hcl.parseHCL(content);
 
-  deps.forEach((dep) => {
-    switch (dep.managerData?.terraformDependencyType) {
-      case 'required_providers':
-        analyzeTerraformRequiredProvider(dep, locks);
-        break;
-      case 'provider':
-        analyzeTerraformProvider(dep, locks);
-        break;
-      case 'module':
-        analyseTerraformModule(dep);
-        break;
-      case 'resource':
-        analyseTerraformResource(dep);
-        break;
-      case 'terraform_version':
-        analyseTerraformVersion(dep);
-        break;
-      /* istanbul ignore next */
-      default:
-    }
+  const locks = await extractLocksForPackageFile(fileName);
+
+  for (const extractor of dependencyExtractors) {
+    const deps = extractor.extract(hclMap, locks);
+    dependencies.push(...deps);
+  }
 
-    delete dep.managerData;
-  });
-  if (deps.some((dep) => dep.skipReason !== 'local')) {
-    return { deps };
+  dependencies.forEach((value) => delete value.managerData);
+  if (dependencies.some((dep) => dep.skipReason !== 'local')) {
+    return { deps: dependencies };
   }
   return null;
 }

lib/modules/manager/terraform/extractors.ts

@@ -0,0 +1,20 @@
+import type { DependencyExtractor } from './base';
+import { ModuleExtractor } from './extractors/others/modules';
+import { ProvidersExtractor } from './extractors/others/providers';
+import { GenericDockerImageRef } from './extractors/resources/generic-docker-image-ref';
+import { HelmReleaseExtractor } from './extractors/resources/helm-release';
+import { TerraformWorkspaceExtractor } from './extractors/resources/terraform-workspace';
+import { RequiredProviderExtractor } from './extractors/terraform-block/required-provider';
+import { TerraformVersionExtractor } from './extractors/terraform-block/terraform-version';
+
+const resourceExtractors: DependencyExtractor[] = [];
+
+export default resourceExtractors;
+
+resourceExtractors.push(new HelmReleaseExtractor());
+resourceExtractors.push(new GenericDockerImageRef());
+resourceExtractors.push(new TerraformWorkspaceExtractor());
+resourceExtractors.push(new RequiredProviderExtractor());
+resourceExtractors.push(new TerraformVersionExtractor());
+resourceExtractors.push(new ProvidersExtractor());
+resourceExtractors.push(new ModuleExtractor());