Update tf provider parsing for submodules (#2691)

Enable recursive parsing of AWS provider configurations to support
providers defined within Terraform submodules.

---
Linear Issue:
[ENG-1393](https://linear.app/overmind/issue/ENG-1393/support-submodule-providers)

<a
href="https://cursor.com/background-agent?bcId=bc-70333602-3079-47bf-9c58-7751a694c64b"><picture><source
media="(prefers-color-scheme: dark)"
srcset="https://cursor.com/open-in-cursor-dark.svg"><source
media="(prefers-color-scheme: light)"
srcset="https://cursor.com/open-in-cursor-light.svg"><img alt="Open in
Cursor"
src="https://cursor.com/open-in-cursor.svg"></picture></a>&nbsp;<a
href="https://cursor.com/agents?id=bc-70333602-3079-47bf-9c58-7751a694c64b"><picture><source
media="(prefers-color-scheme: dark)"
srcset="https://cursor.com/open-in-web-dark.svg"><source
media="(prefers-color-scheme: light)"
srcset="https://cursor.com/open-in-web-light.svg"><img alt="Open in Web"
src="https://cursor.com/open-in-web.svg"></picture></a>

---------

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
GitOrigin-RevId: 3a1d807ce26bd05af7646948a0407d3a82820605

Author: 2 people

cmd/explore.go

@@ -2,10 +2,11 @@ package cmd
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"os"
-	"path/filepath"
 	"slices"
+	"strings"
 
 	"atomicgo.dev/keyboard"
 	"atomicgo.dev/keyboard/keys"
@@ -24,6 +25,7 @@ import (
 	log "github.com/sirupsen/logrus"
 	"github.com/sourcegraph/conc/pool"
 	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
 	"golang.org/x/oauth2"
 )
 
@@ -52,6 +54,9 @@ For GCP, ensure you have appropriate permissions (roles/browser or equivalent) t
 func StartLocalSources(ctx context.Context, oi sdp.OvermindInstance, token *oauth2.Token, tfArgs []string, failOverToDefaultLoginCfg bool) (func(), error) {
 	var err error
 
+	// Default to recursive search unless --no-recursion is set
+	tfRecursive := !viper.GetBool("no-recursion")
+
 	multi := pterm.DefaultMultiPrinter
 	_, _ = multi.Start()
 	defer func() {
@@ -67,22 +72,31 @@ func StartLocalSources(ctx context.Context, oi sdp.OvermindInstance, token *oaut
 
 	p := pool.NewWithResults[[]*discovery.Engine]().WithErrors()
 
-	tfFiles, err := filepath.Glob(filepath.Join(".", "*.tf"))
+	// find all the terraform files
+	tfFiles, err := tfutils.FindTerraformFiles(".", tfRecursive)
 	if err != nil {
+		// we only error if there is a filesystem error, 0 files is handled below
 		return nil, err
 	}
 
+	// if no terraform files are found, return an error
 	if len(tfFiles) == 0 && !failOverToDefaultLoginCfg {
 		currentDir, _ := os.Getwd()
-		return nil, fmt.Errorf(`No Terraform configuration files found in %s
-
-The Overmind CLI requires access to Terraform configuration files (.tf files) to discover and authenticate with cloud providers. Without Terraform configuration, the CLI cannot determine which cloud resources to interrogate.
-
-To resolve this issue:
-- Ensure you're running the command from a directory containing Terraform files (.tf files)
-- Or create Terraform configuration files that define your cloud providers
-
-For more information about Terraform configuration, visit: https://developer.hashicorp.com/terraform/language`, currentDir)
+		msgLines := []string{
+			fmt.Sprintf("No Terraform configuration files found in %s", currentDir),
+			"",
+			"The Overmind CLI requires access to Terraform configuration files (.tf files) to discover and authenticate with cloud providers. Without Terraform configuration, the CLI cannot determine which cloud resources to interrogate.",
+			"",
+			"To resolve this issue:",
+			"- Ensure you're running the command from a directory containing Terraform files (.tf files)",
+			"- Or create Terraform configuration files that define your cloud providers",
+			"",
+		}
+		if !tfRecursive {
+			msgLines = append(msgLines, "- Or remove --no-recursion to scan subdirectories for Terraform stacks")
+		}
+		msgLines = append(msgLines, "For more information about Terraform configuration, visit: https://developer.hashicorp.com/terraform/language")
+		return nil, errors.New(strings.Join(msgLines, "\n"))
 	}
 
 	stdlibSpinner, _ := pterm.DefaultSpinner.WithWriter(multi.NewWriter()).Start("Starting stdlib source engine")
@@ -129,7 +143,7 @@ For more information about Terraform configuration, visit: https://developer.has
 			return nil, fmt.Errorf("failed to load variables from the environment: %w", err)
 		}
 
-		awsProviders, err := tfutils.ParseAWSProviders(".", tfEval)
+		awsProviders, err := tfutils.ParseAWSProviders(".", tfEval, tfRecursive)
 		if err != nil {
 			awsSpinner.Fail("Failed to parse AWS providers")
 			return nil, fmt.Errorf("failed to parse AWS providers: %w", err)
@@ -217,7 +231,7 @@ For more information about Terraform configuration, visit: https://developer.has
 			return nil, fmt.Errorf("failed to load variables from the environment for GCP: %w", err)
 		}
 
-		gcpProviders, err := tfutils.ParseGCPProviders(".", tfEval)
+		gcpProviders, err := tfutils.ParseGCPProviders(".", tfEval, tfRecursive)
 		if err != nil {
 			gcpSpinner.Fail("Failed to parse GCP providers")
 			return nil, fmt.Errorf("failed to parse GCP providers: %w", err)
@@ -404,6 +418,8 @@ func init() {
 	rootCmd.AddCommand(exploreCmd)
 
 	addAPIFlags(exploreCmd)
+	// flag to opt-out of recursion and only scan the current folder for *.tf files
+	exploreCmd.PersistentFlags().Bool("no-recursion", false, "Only scan the current directory for Terraform files (non-recursive).")
 }
 
 // unifiedGCPConfigs collates the given GCP configs by project ID.

tfutils/aws_config.go

@@ -472,17 +472,12 @@ type ProviderResult struct {
 	FilePath string
 }
 
-// Parses AWS provider config from all terraform files in the given directory,
-// without recursion as we don't yet support providers in submodules. Returns a
-// list of AWS providers and a list of files that were parsed. This will return
-// an error only if there was an error loading the files. ProviderResults will
-// be returned for:
-//
-// * Files that could not be parsed at all (just an error)
-// * Files that contained an AWS provider that we couldn't fully evaluate (with an error)
-// * Files that contained an AWS provider that we could fully evaluate (with no error)
-func ParseAWSProviders(terraformDir string, evalContext *hcl.EvalContext) ([]ProviderResult, error) {
-	files, err := filepath.Glob(filepath.Join(terraformDir, "*.tf"))
+// ParseAWSProviders scans for .tf files and extracts AWS provider configurations.
+// The search behavior is controlled by the recursive flag: when false, only the
+// provided directory is scanned via a simple glob; when true, the directory is
+// walked recursively while skipping dot-directories (e.g., .terraform).
+func ParseAWSProviders(terraformDir string, evalContext *hcl.EvalContext, recursive bool) ([]ProviderResult, error) {
+	files, err := FindTerraformFiles(terraformDir, recursive)
 	if err != nil {
 		return nil, err
 	}
@@ -549,6 +544,38 @@ func ParseAWSProviders(terraformDir string, evalContext *hcl.EvalContext) ([]Pro
 	return results, nil
 }
 
+// FindTerraformFiles returns a list of Terraform files under terraformDir.
+// When recursive is false, it uses a simple glob for "*.tf" in the directory.
+// When recursive is true, it walks the directory tree and collects .tf files,
+// skipping any dot-prefixed subdirectories (e.g., .terraform).
+func FindTerraformFiles(terraformDir string, recursive bool) ([]string, error) {
+	if !recursive {
+		return filepath.Glob(filepath.Join(terraformDir, "*.tf"))
+	}
+	files := []string{}
+	err := filepath.Walk(terraformDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		// If this is a subdirectory starting with a dot, skip it entirely
+		if info.IsDir() && path != terraformDir && strings.HasPrefix(filepath.Base(path), ".") {
+			return filepath.SkipDir
+		}
+		if info.IsDir() {
+			return nil
+		}
+		// Only include .tf files
+		if strings.HasSuffix(path, ".tf") {
+			files = append(files, path)
+		}
+		return nil
+	})
+	if err != nil {
+		return nil, fmt.Errorf("error walking directory %s: %w", terraformDir, err)
+	}
+	return files, nil
+}
+
 // ConfigFromProvider creates an aws.Config from an AWSProvider that uses the
 // provided HTTP client. This client will be modified with proxy settings if
 // they are present in the provider.