feat: support import blocks by root module deployment

pkg/deployer/terraform/deployer.go

@@ -179,12 +179,31 @@ func (d Deployer) createK8sJob(ctx context.Context, mc model.ClientSet, opts cre
 		return err
 	}
 
+	resource, err := mc.ResourceRuns().
+		QueryResource(opts.ResourceRun).
+		WithTemplate().
+		WithResourceDefinitionMatchingRule(func(query *model.ResourceDefinitionMatchingRuleQuery) {
+			query.WithTemplate()
+		}).
+		Only(ctx)
+	if err != nil {
+		return err
+	}
+
+	var template *model.TemplateVersion
+	if resource.TemplateID != nil {
+		template = resource.Edges.Template
+	} else {
+		template = resource.Edges.ResourceDefinitionMatchingRule.Edges.Template
+	}
+
 	// Create a deployment job.
 	jobOpts := JobCreateOptions{
 		Type:        opts.Type,
 		Image:       jobImage,
 		Env:         jobEnv,
 		DockerMode:  localEnvironmentMode == "docker",
+		Template:    template,
 		ResourceRun: opts.ResourceRun,
 		ServerURL:   secretOpts.SeverULR,
 		Token:       secretOpts.Token,

pkg/deployer/terraform/jobctrl.go

@@ -19,6 +19,7 @@ import (
 	"github.com/seal-io/walrus/pkg/dao/types/object"
 	opk8s "github.com/seal-io/walrus/pkg/operator/k8s"
 	"github.com/seal-io/walrus/pkg/operator/k8s/kube"
+	"github.com/seal-io/walrus/pkg/vcs"
 	"github.com/seal-io/walrus/utils/log"
 	"github.com/seal-io/walrus/utils/pointer"
 )
@@ -31,6 +32,7 @@ type JobCreateOptions struct {
 	Env        []corev1.EnvVar
 	DockerMode bool
 
+	Template    *model.TemplateVersion
 	ResourceRun *model.ResourceRun
 	Token       string
 	ServerURL   string
@@ -137,11 +139,24 @@ func CreateSecret(ctx context.Context, clientSet *kubernetes.Clientset, name str
 
 // getPodTemplate returns a pod template for deployment.
 func getPodTemplate(configName string, opts JobCreateOptions) corev1.PodTemplateSpec {
+	repo, err := vcs.ParseURLToRepo(opts.Template.Source)
+	if err != nil {
+		return corev1.PodTemplateSpec{}
+	}
+
 	var (
 		command       = []string{"/bin/sh", "-c"}
-		deployCommand = fmt.Sprintf("cp %s/main.tf main.tf && ", _secretMountPath)
+		deployCommand = fmt.Sprintf("git clone %s /var/terraform/workspace &&", repo.Link)
 	)
 
+	deployCommand += fmt.Sprintf("git checkout %s && ", opts.Template.Version)
+
+	if repo.SubPath != "" {
+		deployCommand += fmt.Sprintf("cd %s && ", repo.SubPath)
+	}
+
+	deployCommand += fmt.Sprintf("cp %s/main.tf override.tf && ", _secretMountPath)
+
 	switch opts.Type {
 	case types.RunTaskTypePlan:
 		deployCommand += getPlanCommands(opts.ResourceRun, opts)

pkg/resourceruns/config/common.go

@@ -8,7 +8,6 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 
 	"github.com/seal-io/walrus/pkg/dao/model"
-	"github.com/seal-io/walrus/pkg/dao/types"
 	"github.com/seal-io/walrus/pkg/terraform/config"
 	"github.com/seal-io/walrus/utils/json"
 )
@@ -41,7 +40,7 @@ func updateOutputWithVariables(variables model.Variables, moduleConfig *config.M
 		variableOpts[s.Name] = s.Sensitive
 
 		if s.Sensitive {
-			encryptVariableNames.Insert(_variablePrefix + s.Name)
+			encryptVariableNames.Insert(s.Name)
 		}
 	}
 
@@ -86,23 +85,3 @@ func updateOutputWithVariables(variables model.Variables, moduleConfig *config.M
 
 	return variableOpts, nil
 }
-
-func getVarConfigOptions(
-	variables model.Variables,
-	resourceOutputs map[string]types.OutputValue,
-) config.CreateOptions {
-	varsConfigOpts := config.CreateOptions{
-		Attributes: map[string]any{},
-	}
-
-	for _, v := range variables {
-		varsConfigOpts.Attributes[_variablePrefix+v.Name] = v.Value
-	}
-
-	// Setup resource outputs.
-	for n, v := range resourceOutputs {
-		varsConfigOpts.Attributes[_resourcePrefix+n] = v.Value
-	}
-
-	return varsConfigOpts
-}

pkg/resourceruns/config/parse.go

@@ -22,21 +22,13 @@ import (
 	"github.com/seal-io/walrus/utils/json"
 )
 
-const (
-	// _variablePrefix the prefix of the variable name.
-	_variablePrefix = "_walrus_var_"
-
-	// _resourcePrefix the prefix of the resource output name.
-	_resourcePrefix = "_walrus_res_"
-)
-
 // _interpolationReg is the regular expression for matching non-reference or non-variable expressions.
 // Reference: https://developer.hashicorp.com/terraform/language/expressions/strings#escape-sequences-1
 // To handle escape sequences, ${xxx} is converted to $${xxx}.
 // If there are more than two consecutive $ symbols, like $${xxx}, they are further converted to $$${xxx}.
 // During Terraform processing, $${} is ultimately transformed back to ${};
 // this interpolation is used to ensure a WYSIWYG user experience.
-var _interpolationReg = regexp.MustCompile(`\$\{((var\.)?([^.}]+)(?:\.([^.}]+))?)[^\}]*\}`)
+var _interpolationReg = regexp.MustCompile(`\$\{((var\.|res\.)?([^.}]+)(?:\.([^.}]+))?)[^\}]*\}`)
 
 type RunOpts struct {
 	ResourceRun *model.ResourceRun
@@ -64,7 +56,7 @@ func ParseModuleAttributes(
 
 	attrs, templateVariables, dependencyResourceOutputs, err = parseAttributeReplace(attributes, replaced)
 	if err != nil {
-		return
+		return nil, nil, nil, err
 	}
 
 	// If a resource run has variables that inherit from cloned run, use them directly.
@@ -82,7 +74,32 @@ func ParseModuleAttributes(
 		}
 	}
 
+	vars := make(map[string]*model.Variable)
+	for _, v := range variables {
+		vars[v.Name] = v
+	}
+
 	if !onlyValidated {
+		// Replace variables and resource references.
+		bs, err := json.Marshal(attributes)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+
+		bs = interpolation.VariableReg.ReplaceAllFunc(bs, func(match []byte) []byte {
+			m := interpolation.VariableReg.FindSubmatch(match)
+
+			if len(m) != 2 {
+				return match
+			}
+
+			if v, ok := vars[string(m[1])]; ok {
+				return []byte(v.Value)
+			}
+
+			return match
+		})
+
 		dependOutputMap := toDependOutputMap(dependencyResourceOutputs)
 
 		outputs, err = getResourceDependencyOutputsByID(
@@ -101,6 +118,31 @@ func ParseModuleAttributes(
 					opts.ResourceName, outputName)
 			}
 		}
+
+		bs = interpolation.ResourceReg.ReplaceAllFunc(bs, func(match []byte) []byte {
+			m := interpolation.ResourceReg.FindSubmatch(match)
+
+			if len(m) != 3 {
+				return match
+			}
+
+			if v, ok := outputs[fmt.Sprintf("%s_%s", m[1], m[2])]; ok {
+				var str string
+				err := json.Unmarshal(v.Value, &str)
+				if err != nil {
+					return v.Value
+				}
+				return []byte(str)
+			}
+
+			return match
+		})
+
+		attrs = make(map[string]any)
+		err = json.Unmarshal(bs, &attrs)
+		if err != nil {
+			return nil, nil, nil, err
+		}
 	}
 
 	return attrs, variables, outputs, nil
@@ -156,12 +198,6 @@ func parseAttributeReplace(
 		}
 	}
 
-	variableRepl := "${var." + _variablePrefix + "${1}}"
-	bs = interpolation.VariableReg.ReplaceAll(bs, []byte(variableRepl))
-
-	resourceRepl := "${var." + _resourcePrefix + "${1}_${2}}"
-	bs = interpolation.ResourceReg.ReplaceAll(bs, []byte(resourceRepl))
-
 	// Replace interpolation from ${} to $${} to avoid escape sequences.
 	bs = _interpolationReg.ReplaceAllFunc(bs, func(match []byte) []byte {
 		m := _interpolationReg.FindSubmatch(match)
@@ -171,7 +207,7 @@ func parseAttributeReplace(
 		}
 
 		// If it is a variable or resource reference, do not replace.
-		if string(m[2]) == "var." {
+		if string(m[2]) == "var." || string(m[2]) == "res." {
 			return match
 		}
 

pkg/resourceruns/config/parse_test.go

@@ -96,7 +96,7 @@ func TestParseAttributeReplace(t *testing.T) {
 			expectedVariableNames:   []string{},
 			expectedResourceOutputs: []string{"res_foo_bar"},
 			expectedAttributes: map[string]any{
-				"foo": "${var._walrus_res_foo_bar}",
+				"foo": "${res.foo.bar}",
 			},
 			expectedError: false,
 		},
@@ -110,8 +110,8 @@ func TestParseAttributeReplace(t *testing.T) {
 			expectedVariableNames:   []string{"foo"},
 			expectedResourceOutputs: []string{"res_foo1_bar", "res_foo2_bar"},
 			expectedAttributes: map[string]any{
-				"foo": "${var._walrus_var_foo}",
-				"bar": "${var._walrus_res_foo1_bar}-${var._walrus_res_foo2_bar}",
+				"foo": "${var.foo}",
+				"bar": "${res.foo1.bar}-${res.foo2.bar}",
 			},
 			expectedError: false,
 		},
@@ -128,9 +128,9 @@ func TestParseAttributeReplace(t *testing.T) {
 			expectedVariableNames:   []string{"foo"},
 			expectedResourceOutputs: []string{"res_foo1_bar", "res_foo2_bar"},
 			expectedAttributes: map[string]any{
-				"foo":    "${var._walrus_var_foo}",
-				"bar":    "${var._walrus_res_foo1_bar}-${var._walrus_res_foo2_bar}",
-				"baz":    "${var._walrus_var_foo}-${var._walrus_res_foo1_bar}-${var._walrus_res_foo2_bar}",
+				"foo":    "${var.foo}",
+				"bar":    "${res.foo1.bar}-${res.foo2.bar}",
+				"baz":    "${var.foo}-${res.foo1.bar}-${res.foo2.bar}",
 				"qux":    "$${MYSQL_DATABASE}",
 				"double": "$$${ENV_PORT}", // Terraform will replace $$ with $.
 			},
@@ -153,8 +153,8 @@ func TestParseAttributeReplace(t *testing.T) {
 			expectedResourceOutputs: []string{"res_foo1_bar", "res_foo2_bar"},
 			expectedAttributes: map[string]any{
 				"foo": []any{
-					"${var._walrus_var_foo}",
-					"${var._walrus_res_foo1_bar}-${var._walrus_res_foo2_bar}",
+					"${var.foo}",
+					"${res.foo1.bar}-${res.foo2.bar}",
 				},
 				"ENV": []any{
 					"$${ENV_PORT}",

pkg/resourceruns/config/tfconfig.go

@@ -95,8 +95,6 @@ func (c *TerraformConfigurator) LoadAll(
 		return nil, err
 	}
 
-	moduleConfig.Attributes = attrs
-
 	// Update output sensitive with variables.
 	wrapVariables, err := updateOutputWithVariables(variables, moduleConfig)
 	if err != nil {
@@ -130,18 +128,16 @@ func (c *TerraformConfigurator) LoadAll(
 				SecretMonthPath:       opts.SecretMountPath,
 				ConnectorSeparator:    parser.ConnectorSeparator,
 			},
-			ModuleOptions: &config.ModuleOptions{
-				ModuleConfigs: []*config.ModuleConfig{moduleConfig},
-			},
 			VariableOptions: &config.VariableOptions{
-				VariablePrefix:    _variablePrefix,
-				ResourcePrefix:    _resourcePrefix,
 				Variables:         wrapVariables,
 				DependencyOutputs: dependencyOutputs,
+				Attributes:        moduleConfig.Attributes,
 			},
 			OutputOptions: moduleConfig.Outputs,
 		},
-		config.FileVars: getVarConfigOptions(variables, dependencyOutputs),
+		config.FileVars: {
+			Attributes: attrs,
+		},
 	}
 
 	inputConfigs := make(map[string]types.ResourceRunConfigData, len(tfCreateOpts))

pkg/resourcestate/state.go

@@ -2,6 +2,7 @@ package resourcestate
 
 import (
 	"context"
+	"strings"
 
 	"github.com/seal-io/walrus/pkg/dao/model"
 	"github.com/seal-io/walrus/pkg/dao/model/resourcestate"
@@ -19,31 +20,35 @@ func GetDependencyOutputs(
 ) (map[string]types.OutputValue, error) {
 	states, err := client.ResourceStates().Query().
 		Where(resourcestate.ResourceIDIn(dependencyResourceIDs...)).
+		WithResource().
 		All(ctx)
 	if err != nil {
 		return nil, err
 	}
 
-	outputs := make(map[string]types.OutputValue)
-
 	var p parser.StateParser
 
+	// Get the outputs of the dependency resources.
+	resToOutputs := make(map[string]map[string]types.OutputValue)
 	for _, s := range states {
-		osm, err := p.GetOutputMap(s.Data)
+		resToOutputs[s.Edges.Resource.Name], err = p.GetOutputMap(s.Data)
 		if err != nil {
 			return nil, err
 		}
+	}
+
+	outputs := make(map[string]types.OutputValue)
+
+	for k := range dependOutputs {
+		split := strings.Split(k, "_")
+		res, output := split[0], split[1]
 
-		for n, o := range osm {
-			if _, ok := dependOutputs[n]; !ok {
-				continue
-			}
+		o := resToOutputs[res][output]
 
-			outputs[n] = types.OutputValue{
-				Value:     o.Value,
-				Type:      o.Type,
-				Sensitive: o.Sensitive,
-			}
+		outputs[k] = types.OutputValue{
+			Value:     o.Value,
+			Type:      o.Type,
+			Sensitive: o.Sensitive,
 		}
 	}