WIP: User provider_installation settings with the provider cache

cli/provider_cache.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"net/url"
 	"os"
 	"path/filepath"
 	"regexp"
@@ -18,8 +19,6 @@ import (
 	"github.com/gruntwork-io/terragrunt/shell"
 	"github.com/gruntwork-io/terragrunt/terraform"
 	"github.com/gruntwork-io/terragrunt/terraform/cache"
-	"github.com/gruntwork-io/terragrunt/terraform/cache/controllers"
-	"github.com/gruntwork-io/terragrunt/terraform/cache/handlers"
 	"github.com/gruntwork-io/terragrunt/terraform/cliconfig"
 	"github.com/gruntwork-io/terragrunt/terraform/getproviders"
 	"github.com/gruntwork-io/terragrunt/util"
@@ -29,6 +28,10 @@ import (
 const (
 	// The paths to the automatically generated local CLI configs
 	localCLIFilename = ".terraformrc"
+
+	// The status returned when making a request to the caching provider.
+	// It is needed to prevent further loading of providers by terraform, and at the same time make sure that the request was processed successfully.
+	cacheProviderHTTPStatusCode = http.StatusLocked
 )
 
 var (
@@ -46,14 +49,25 @@ var (
 	//    │ provider registry for registry.terraform.io/snowflake-labs/snowflake: 423
 	//    │ Locked
 	//    ╵
-	HTTPStatusCacheProviderReg = regexp.MustCompile(`(?smi)` + strconv.Itoa(controllers.HTTPStatusCacheProvider) + `.*` + http.StatusText(controllers.HTTPStatusCacheProvider))
+	HTTPStatusCacheProviderReg = regexp.MustCompile(`(?smi)` + strconv.Itoa(cacheProviderHTTPStatusCode) + `.*` + http.StatusText(cacheProviderHTTPStatusCode))
 )
 
 type ProviderCache struct {
 	*cache.Server
+	cliCfg *cliconfig.Config
 }
 
 func InitProviderCacheServer(opts *options.TerragruntOptions) (*ProviderCache, error) {
+	cliCfg, err := cliconfig.LoadUserConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	// for _, m := range cliCfg.ProviderInstallation.Methods {
+	// 	fmt.Println("---------------------", m)
+	// }
+	// os.Exit(1)
+
 	cacheDir, err := util.GetCacheDir()
 	if err != nil {
 		return nil, err
@@ -73,8 +87,8 @@ func InitProviderCacheServer(opts *options.TerragruntOptions) (*ProviderCache, e
 		opts.ProviderCacheToken = uuid.New().String()
 	}
 	// Currently, the cache server only supports the `x-api-key` token.
-	if !strings.HasPrefix(strings.ToLower(opts.ProviderCacheToken), handlers.AuthorizationApiKeyHeaderName+":") {
-		opts.ProviderCacheToken = fmt.Sprintf("%s:%s", handlers.AuthorizationApiKeyHeaderName, opts.ProviderCacheToken)
+	if !strings.HasPrefix(strings.ToLower(opts.ProviderCacheToken), cache.AuthorizationApiKeyHeaderName+":") {
+		opts.ProviderCacheToken = fmt.Sprintf("%s:%s", cache.AuthorizationApiKeyHeaderName, opts.ProviderCacheToken)
 	}
 
 	userProviderDir, err := cliconfig.UserProviderDir()
@@ -90,7 +104,10 @@ func InitProviderCacheServer(opts *options.TerragruntOptions) (*ProviderCache, e
 		cache.WithProviderCacheDir(opts.ProviderCacheDir),
 	)
 
-	return &ProviderCache{Server: cache}, nil
+	return &ProviderCache{
+		Server: cache,
+		cliCfg: cliCfg,
+	}, nil
 }
 
 func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *options.TerragruntOptions, args []string) (*shell.CmdOutput, error) {
@@ -168,31 +185,27 @@ func (cache *ProviderCache) TerraformCommandHook(ctx context.Context, opts *opti
 // 1. If `cacheRequestID` is set, `terraform init` does _not_ use the provider cache directory, the cache server creates a cache for requested providers and returns HTTP status 423. Since for each module we create the CLI config, using `cacheRequestID` we have the opportunity later retrieve from the cache server exactly those cached providers that were requested by `terraform init` using this configuration.
 // 2. If `cacheRequestID` is empty, 'terraform init` uses provider cache directory, the cache server acts as a proxy.
 func (cache *ProviderCache) createLocalCLIConfig(opts *options.TerragruntOptions, filename string, cacheRequestID string) error {
-	cfg, err := cliconfig.LoadUserConfig()
-	if err != nil {
-		return err
-	}
+	cfg := cache.cliCfg.Clone()
 	cfg.PluginCacheDir = ""
 
 	var providerInstallationIncludes []string
 
 	for _, registryName := range opts.ProviderCacheRegistryNames {
 		providerInstallationIncludes = append(providerInstallationIncludes, fmt.Sprintf("%s/*/*", registryName))
 
+		//networkMirrorURL := "https://mirrors.tencent.com/terraform/"
+		//networkMirrorURL := "http://127.0.0.1:7654"
+		networkMirrorURL := ""
+
 		cfg.AddHost(registryName, map[string]string{
-			"providers.v1": fmt.Sprintf("%s/%s/%s/", cache.ProviderURL(), cacheRequestID, registryName),
+			"providers.v1": fmt.Sprintf("%s/%s/%s/%s/", cache.ProviderURL(), url.QueryEscape(networkMirrorURL), cacheRequestID, registryName),
 			// Since Terragrunt Provider Cache only caches providers, we need to route module requests to the original registry.
 			"modules.v1": fmt.Sprintf("https://%s/v1/modules", registryName),
 		})
 	}
 
-	if cacheRequestID != "" {
-		cfg.SetProviderInstallation(
-			nil,
-			cliconfig.NewProviderInstallationDirect(nil, nil),
-		)
-	} else {
-		cfg.SetProviderInstallation(
+	if cacheRequestID == "" {
+		cfg.AddProviderInstallationMethods(
 			cliconfig.NewProviderInstallationFilesystemMirror(opts.ProviderCacheDir, providerInstallationIncludes, nil),
 			cliconfig.NewProviderInstallationDirect(providerInstallationIncludes, nil),
 		)

pkg/log/exported.go

@@ -131,3 +131,9 @@ func Logf(level logrus.Level, format string, args ...interface{}) {
 		logger.Log(level, fmt.Sprintf(format, args...))
 	}
 }
+
+// WithField allocates a new entry and adds a field to it.
+func WithField(key string, value interface{}) *logrus.Entry {
+	return logger.WithField(key, value)
+
+}

terraform/cache/config.go

@@ -4,6 +4,8 @@ import (
 	"net"
 	"strconv"
 	"time"
+
+	"github.com/gruntwork-io/terragrunt/terraform/cache/handlers"
 )
 
 const (
@@ -52,6 +54,13 @@ func WithUserProviderDir(userProviderDir string) Option {
 	}
 }
 
+func WithProviderHandlers(handlers ...handlers.ProviderHandler) Option {
+	return func(cfg Config) Config {
+		cfg.providerHandlers = handlers
+		return cfg
+	}
+}
+
 type Config struct {
 	hostname        string
 	port            int
@@ -60,6 +69,7 @@ type Config struct {
 
 	userProviderDir  string
 	providerCacheDir string
+	providerHandlers []handlers.ProviderHandler
 }
 
 func NewConfig(opts ...Option) *Config {

terraform/cache/controllers/discovery.go

@@ -17,16 +17,18 @@ type Endpointer interface {
 }
 
 type DiscoveryController struct {
+	*router.Router
+
 	Endpointers []Endpointer
 }
 
 // Register implements router.Controller.Register
 func (controller *DiscoveryController) Register(router *router.Router) {
-	router = router.Group(discoveryPath)
+	controller.Router = router.Group(discoveryPath)
 
 	// Discovery Process
 	// https://developer.hashicorp.com/terraform/internals/remote-service-discovery#discovery-process
-	router.GET("/terraform.json", controller.terraformAction)
+	controller.GET("/terraform.json", controller.terraformAction)
 }
 
 func (controller *DiscoveryController) terraformAction(ctx echo.Context) error {

terraform/cache/controllers/downloader.go

@@ -1,43 +1,31 @@
 package controllers
 
 import (
+	"net/http"
 	"net/url"
-	"path"
 
-	"github.com/gruntwork-io/terragrunt/pkg/log"
 	"github.com/gruntwork-io/terragrunt/terraform/cache/handlers"
 	"github.com/gruntwork-io/terragrunt/terraform/cache/models"
 	"github.com/gruntwork-io/terragrunt/terraform/cache/router"
-	"github.com/gruntwork-io/terragrunt/terraform/cache/services"
 	"github.com/labstack/echo/v4"
 )
 
 const (
-	downloadPath         = "/downloads"
-	downloadProviderPath = "/provider"
+	downloadPath = "/downloads"
 )
 
 type DownloaderController struct {
-	ReverseProxy    *handlers.ReverseProxy
-	ProviderService *services.ProviderService
+	*router.Router
 
-	basePath string
-}
-
-// ProviderProxyURL returns URL for using as a cache to download remote archives through this controller with caching
-func (controller *DownloaderController) ProviderProxyURL() *url.URL {
-	cacheURL := *controller.ReverseProxy.ServerURL
-	cacheURL.Path = path.Join(cacheURL.Path, controller.basePath, downloadProviderPath)
-	return &cacheURL
+	ProviderHandlers []handlers.ProviderHandler
 }
 
 // Register implements router.Controller.Register
 func (controller *DownloaderController) Register(router *router.Router) {
-	router = router.Group(downloadPath)
-	controller.basePath = router.Prefix()
+	controller.Router = router.Group(downloadPath)
 
 	// Download provider
-	router.GET(downloadProviderPath+"/:remote_host/:remote_path", controller.downloadProviderAction)
+	controller.GET("/:remote_host/:remote_path", controller.downloadProviderAction)
 }
 
 func (controller *DownloaderController) downloadProviderAction(ctx echo.Context) error {
@@ -46,19 +34,21 @@ func (controller *DownloaderController) downloadProviderAction(ctx echo.Context)
 		remotePath = ctx.Param("remote_path")
 	)
 
-	downloadURL := &url.URL{
+	downloadURL := url.URL{
 		Scheme: "https",
 		Host:   remoteHost,
 		Path:   "/" + remotePath,
 	}
-	provider := models.NewProviderFromDownloadURL(downloadURL.String())
+	provider := &models.Provider{
+		ResponseBody: &models.ResponseBody{
+			DownloadURL: downloadURL.String(),
+		},
+	}
 
-	if cache := controller.ProviderService.GetProviderCache(provider); cache != nil {
-		if filename := cache.Filename(); filename != "" {
-			log.Debugf("Using cached provider %s", cache.Provider)
-			return ctx.File(filename)
+	for _, handler := range controller.ProviderHandlers {
+		if handler.CanHandleProvider(provider) {
+			return handler.Download(ctx, provider)
 		}
 	}
-
-	return controller.ReverseProxy.NewRequest(ctx, downloadURL)
+	return ctx.NoContent(http.StatusNotFound)
 }