fix: Request correct config format (YAML) from provider (#968)

Co-authored-by: Kemal Hadimli <disq@users.noreply.github.com>

cmd/init.go

@@ -133,7 +133,7 @@ func generateYAMLConfig(ctx context.Context, c *console.Client, providers []stri
 	}
 
 	provNode := &yaml.Node{
-		Kind:        yaml.MappingNode,
+		Kind:        yaml.SequenceNode,
 		HeadComment: "provider configurations",
 	}
 
@@ -155,10 +155,18 @@ func generateYAMLConfig(ctx context.Context, c *console.Client, providers []stri
 		}
 
 		provNode.Content = append(provNode.Content, &yaml.Node{
-			Kind:  yaml.ScalarNode,
-			Value: p,
+			Kind: yaml.MappingNode,
+			Content: append([]*yaml.Node{
+				{
+					Kind:  yaml.ScalarNode,
+					Value: "name",
+				},
+				{
+					Kind:  yaml.ScalarNode,
+					Value: p,
+				},
+			}, yCfg.Content[0].Content...),
 		})
-		provNode.Content = append(provNode.Content, yCfg.Content...)
 	}
 
 	nd := struct {

pkg/config/config.go

@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	"github.com/cloudquery/cloudquery/internal/logging"
+	"github.com/cloudquery/cq-provider-sdk/cqproto"
 	"github.com/spf13/viper"
 	"github.com/xo/dburl"
 )
@@ -22,13 +23,18 @@ type Provider struct {
 	MaxParallelResourceFetchLimit uint64   `yaml:"max_parallel_resource_fetch_limit,omitempty" json:"max_parallel_resource_fetch_limit,omitempty" hcl:"max_parallel_resource_fetch_limit"`
 	MaxGoroutines                 uint64   `yaml:"max_goroutines,omitempty" json:"max_goroutines,omitempty" hcl:"max_goroutines"`
 	ResourceTimeout               uint64   `yaml:"resource_timeout,omitempty" json:"resource_timeout,omitempty" hcl:"resource_timeout"`
+
+	// ConfigKeys is only used temporarily for provider-specific configuration when decoding YAML
+	ConfigKeys map[string]interface{} `yaml:",inline"`
 }
 
 type Providers []*Provider
 
 type Config struct {
 	CloudQuery CloudQuery `hcl:"cloudquery,block" yaml:"cloudquery" json:"cloudquery"`
 	Providers  Providers  `hcl:"provider,block" yaml:"providers" json:"providers"`
+
+	format cqproto.ConfigFormat
 }
 
 type CloudQuery struct {
@@ -76,6 +82,10 @@ func (pp Providers) Names() []string {
 	return pNames
 }
 
+func (c Config) Format() cqproto.ConfigFormat {
+	return c.format
+}
+
 func (c Config) GetProvider(name string) (*Provider, error) {
 	for _, p := range c.Providers {
 		if name == p.Alias {

pkg/config/config_parser.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/cloudquery/cloudquery/internal/logging"
+	"github.com/cloudquery/cq-provider-sdk/cqproto"
 	"github.com/cloudquery/cq-provider-sdk/provider/diag"
 	"github.com/hashicorp/hcl/v2"
 	"github.com/hashicorp/hcl/v2/gohcl"
@@ -101,10 +102,43 @@ func ValidateCQBlock(cq *CloudQuery) diag.Diagnostics {
 	return diags
 }
 
+func ProcessValidateProviderBlock(plist []*Provider) (Providers, diag.Diagnostics) {
+	var diags diag.Diagnostics
+	existingProviders := make(map[string]struct{}, len(plist))
+
+	var ret Providers
+
+	for _, v := range plist {
+		if v.Alias != "" {
+			if _, ok := existingProviders[v.Alias]; ok {
+				diags = diags.Add(diag.FromError(fmt.Errorf("provider with alias %s for provider %s already exists, give it a different alias", v.Alias, v.Name), diag.USER, diag.WithSummary("Duplicate Alias")))
+				continue
+			}
+			existingProviders[v.Alias] = struct{}{}
+		} else {
+			if _, ok := existingProviders[v.Name]; ok {
+				diags = diags.Add(diag.FromError(fmt.Errorf("provider with name %s already exists, use alias in provider configuration block", v.Name), diag.USER, diag.WithSummary("Provider Alias Required")))
+				continue
+			}
+			existingProviders[v.Name] = struct{}{}
+			v.Alias = v.Name
+		}
+		var err error
+		v.Configuration, err = yaml.Marshal(v.ConfigKeys)
+		if err != nil {
+			diags = diags.Add(diag.FromError(err, diag.INTERNAL, diag.WithSummary("ConfigKeys marshal failed")))
+			continue
+		}
+		ret = append(ret, v)
+	}
+
+	return ret, diags
+}
+
 func decodeConfigYAML(r io.Reader) (*Config, diag.Diagnostics) {
 	var yc struct {
-		CloudQuery CloudQuery           `yaml:"cloudquery" json:"cloudquery"`
-		Providers  map[string]*Provider `yaml:"providers" json:"providers"`
+		CloudQuery CloudQuery  `yaml:"cloudquery" json:"cloudquery"`
+		Providers  []*Provider `yaml:"providers" json:"providers"`
 	}
 
 	lgc := logging.GlobalConfig
@@ -136,17 +170,12 @@ func decodeConfigYAML(r io.Reader) (*Config, diag.Diagnostics) {
 
 	c := &Config{
 		CloudQuery: yc.CloudQuery,
-	}
-	for k := range yc.Providers {
-		v := yc.Providers[k]
-		v.Name = k
-		if v.Alias == "" {
-			v.Alias = v.Name
-		}
-		c.Providers = append(c.Providers, v)
+		format:     cqproto.ConfigYAML,
 	}
 
-	diags := ValidateCQBlock(&c.CloudQuery)
+	var diags diag.Diagnostics
+	c.Providers, diags = ProcessValidateProviderBlock(yc.Providers)
+	diags = diags.Add(ValidateCQBlock(&c.CloudQuery))
 	if diags.HasErrors() {
 		return nil, diags
 	}
@@ -156,7 +185,9 @@ func decodeConfigYAML(r io.Reader) (*Config, diag.Diagnostics) {
 
 func (p *Parser) decodeConfigHCL(body hcl.Body, diags diag.Diagnostics) (*Config, diag.Diagnostics) {
 	existingProviders := make(map[string]bool)
-	config := &Config{}
+	config := &Config{
+		format: cqproto.ConfigHCL,
+	}
 
 	content, contentDiags := body.Content(configSchemaHCL)
 	diags = diags.Add(hclToSdkDiags(contentDiags))

pkg/core/fetch.go

@@ -77,8 +77,9 @@ type FetchResponse struct {
 }
 
 type ProviderInfo struct {
-	Provider registry.Provider
-	Config   *config.Provider
+	Provider     registry.Provider
+	Config       *config.Provider
+	ConfigFormat cqproto.ConfigFormat
 }
 
 // FetchOptions is provided to the Client to execute a fetch on one or more providers
@@ -274,6 +275,7 @@ func runProviderFetch(ctx context.Context, pm *plugin.Manager, info ProviderInfo
 		},
 		Config:      cfg.Configuration,
 		ExtraFields: opts.ExtraFields,
+		Format:      info.ConfigFormat,
 	})
 	if err != nil {
 		pLog.Error().Err(err).Msg("failed to configure provider")

pkg/core/provider.go

@@ -19,6 +19,7 @@ type GetProviderConfigOptions struct {
 type TestOptions struct {
 	Connection   cqproto.ConnectionDetails
 	Config       []byte
+	ConfigFormat cqproto.ConfigFormat
 	CreationInfo *plugin.CreationOptions
 }
 
@@ -68,6 +69,7 @@ func Test(ctx context.Context, pm *plugin.Manager, opts TestOptions) (bool, erro
 		CloudQueryVersion: Version,
 		Connection:        opts.Connection,
 		Config:            opts.Config,
+		Format:            opts.ConfigFormat,
 	})
 	if err != nil {
 		return false, fmt.Errorf("provider test connection failed. Reason: %w", err)

pkg/ui/console/client.go

@@ -208,7 +208,7 @@ func (c Client) Fetch(ctx context.Context) (*core.FetchResponse, diag.Diagnostic
 			printDiagnostics("Fetch", &diags, viper.GetBool("redact-diags"), viper.GetBool("verbose"))
 			return nil, diags
 		}
-		providers[i] = core.ProviderInfo{Provider: rp, Config: p}
+		providers[i] = core.ProviderInfo{Provider: rp, Config: p, ConfigFormat: c.cfg.Format()}
 	}
 	result, diags := core.Fetch(ctx, c.StateManager, c.Storage, c.PluginManager, &core.FetchOptions{
 		UpdateCallback: fetchCallback,