feat: Support including deployment projects from git

pkg/deployment/deployment_collection.go

@@ -79,7 +79,7 @@ func (c *DeploymentCollection) collectDeployments(project *DeploymentProject, in
 	var ret []*DeploymentItem
 
 	for i, diConfig := range project.Config.Deployments {
-		if diConfig.Include != nil {
+		if diConfig.Include != nil || diConfig.Git != nil {
 			includedProject, ok := project.includes[i]
 			if !ok {
 				panic(fmt.Sprintf("Did not find find index %d in project.includes", i))

pkg/deployment/deployment_project.go

@@ -146,15 +146,26 @@ func (p *DeploymentProject) checkDeploymentDirs() error {
 
 func (p *DeploymentProject) loadIncludes() error {
 	for i, inc := range p.Config.Deployments {
-		if inc.Include == nil {
+		var err error
+		var newProject *DeploymentProject
+
+		if inc.Include != nil {
+			newProject, err = p.loadLocalInclude(p.rootDir, filepath.Join(p.relDir, *inc.Include), inc.Vars)
+			if err != nil {
+				return err
+			}
+		} else if inc.Git != nil {
+			cloneDir, err := p.ctx.GRC.GetClonedDir(inc.Git.Url, inc.Git.Ref, true, true, true)
+			if err != nil {
+				return err
+			}
+			newProject, err = p.loadLocalInclude(cloneDir, inc.Git.SubDir, inc.Vars)
+			if err != nil {
+				return err
+			}
+		} else {
 			continue
 		}
-
-		newProject, err := p.loadLocalInclude(p.rootDir, filepath.Join(p.relDir, *inc.Include), inc.Vars)
-		if err != nil {
-			return err
-		}
-
 		newProject.parentProjectInclude = inc
 		p.includes[i] = newProject
 	}

pkg/deployment/shared_context.go

@@ -2,13 +2,15 @@ package deployment
 
 import (
 	"context"
+	"github.com/kluctl/kluctl/v2/pkg/git"
 	"github.com/kluctl/kluctl/v2/pkg/k8s"
 	"github.com/kluctl/kluctl/v2/pkg/vars"
 )
 
 type SharedContext struct {
 	Ctx        context.Context
 	K          *k8s.K8sCluster
+	GRC        *git.MirroredGitRepoCollection
 	VarsLoader *vars.VarsLoader
 
 	RenderDir                         string

pkg/git/repo_collection.go

@@ -5,6 +5,11 @@ import (
 	"fmt"
 	"github.com/kluctl/kluctl/v2/pkg/git/auth"
 	git_url "github.com/kluctl/kluctl/v2/pkg/git/git-url"
+	"github.com/kluctl/kluctl/v2/pkg/utils"
+	"io/ioutil"
+	"os"
+	"path"
+	"path/filepath"
 	"sync"
 	"time"
 )
@@ -19,6 +24,7 @@ type MirroredGitRepoCollection struct {
 
 type entry struct {
 	mr          *MirroredGitRepo
+	clonedDirs  map[string]string
 	updateMutex sync.Mutex
 }
 
@@ -36,6 +42,10 @@ func (g *MirroredGitRepoCollection) Clear() {
 	defer g.mutex.Unlock()
 
 	for _, e := range g.repos {
+		for _, path := range e.clonedDirs {
+			_ = os.RemoveAll(path)
+		}
+
 		if e.mr.IsLocked() {
 			_ = e.mr.Unlock()
 		}
@@ -44,7 +54,7 @@ func (g *MirroredGitRepoCollection) Clear() {
 	g.repos = map[string]*entry{}
 }
 
-func (g *MirroredGitRepoCollection) GetMirroredGitRepo(url git_url.GitUrl, allowCreate bool, lockRepo bool, update bool) (*MirroredGitRepo, error) {
+func (g *MirroredGitRepoCollection) getEntry(url git_url.GitUrl, allowCreate bool, lockRepo bool, update bool) (*entry, error) {
 	e, err := func() (*entry, error) {
 		g.mutex.Lock()
 		defer g.mutex.Unlock()
@@ -59,7 +69,8 @@ func (g *MirroredGitRepoCollection) GetMirroredGitRepo(url git_url.GitUrl, allow
 				return nil, err
 			}
 			e = &entry{
-				mr: mr,
+				mr:         mr,
+				clonedDirs: map[string]string{},
 			}
 			g.repos[url.NormalizedRepoKey()] = e
 
@@ -90,5 +101,48 @@ func (g *MirroredGitRepoCollection) GetMirroredGitRepo(url git_url.GitUrl, allow
 		}
 	}
 
+	return e, nil
+}
+
+func (g *MirroredGitRepoCollection) GetMirroredGitRepo(url git_url.GitUrl, allowCreate bool, lockRepo bool, update bool) (*MirroredGitRepo, error) {
+	e, err := g.getEntry(url, allowCreate, lockRepo, update)
+	if err != nil {
+		return nil, err
+	}
 	return e.mr, nil
 }
+
+func (g *MirroredGitRepoCollection) GetClonedDir(url git_url.GitUrl, ref string, allowCreate bool, lockRepo bool, update bool) (string, error) {
+	e, err := g.getEntry(url, allowCreate, lockRepo, update)
+	if err != nil {
+		return "", err
+	}
+
+	e.updateMutex.Lock()
+	defer e.updateMutex.Unlock()
+
+	p, ok := e.clonedDirs[ref]
+	if ok {
+		return p, nil
+	}
+
+	tmpDir := filepath.Join(utils.GetTmpBaseDir(), "git-cloned")
+	err = os.MkdirAll(tmpDir, 0700)
+	if err != nil {
+		return "", err
+	}
+
+	repoName := path.Base(url.Normalize().Path)
+	p, err = ioutil.TempDir(tmpDir, repoName+"-"+ref+"-")
+	if err != nil {
+		return "", err
+	}
+
+	err = e.mr.CloneProject(ref, p)
+	if err != nil {
+		return "", err
+	}
+
+	e.clonedDirs[ref] = p
+	return p, nil
+}

pkg/kluctl_project/target_context.go

@@ -77,6 +77,7 @@ func (p *LoadedKluctlProject) NewTargetContext(ctx context.Context, targetName s
 	dctx := deployment.SharedContext{
 		Ctx:                               ctx,
 		K:                                 k,
+		GRC:                               p.GRC,
 		VarsLoader:                        varsLoader,
 		RenderDir:                         renderOutputDir,
 		SealedSecretsDir:                  p.sealedSecretsDir,

pkg/types/deployment.go

@@ -8,6 +8,7 @@ import (
 type DeploymentItemConfig struct {
 	Path             *string                  `yaml:"path,omitempty"`
 	Include          *string                  `yaml:"include,omitempty"`
+	Git              *GitProject              `yaml:"git,omitempty"`
 	Tags             []string                 `yaml:"tags,omitempty"`
 	Barrier          bool                     `yaml:"barrier,omitempty"`
 	WaitReadiness    bool                     `yaml:"waitReadiness,omitempty"`
@@ -20,8 +21,18 @@ type DeploymentItemConfig struct {
 
 func ValidateDeploymentItemConfig(sl validator.StructLevel) {
 	s := sl.Current().Interface().(DeploymentItemConfig)
-	if s.Path != nil && s.Include != nil {
-		sl.ReportError(s, "path", "Path", "path and include can not be set at the same time", "")
+	cnt := 0
+	if s.Path != nil {
+		cnt += 1
+	}
+	if s.Include != nil {
+		cnt += 1
+	}
+	if s.Git != nil {
+		cnt += 1
+	}
+	if cnt > 1 {
+		sl.ReportError(s, "self", "self", "only one of path, include and git can be set at the same time", "")
 	}
 	if s.Path == nil && s.WaitReadiness {
 		sl.ReportError(s, "waitReadiness", "WaitReadiness", "only kustomize deployments are allowed to have waitReadiness set", "")