refactor: generic k8s resource status getter and common repo

internal/pkg/plugin/argocdapp/state.go

@@ -6,7 +6,7 @@ import (
 	"github.com/cenkalti/backoff"
 
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller"
-	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/util"
+	"github.com/devstream-io/devstream/pkg/util/k8s"
 )
 
 func getStaticState(options plugininstaller.RawOptions) (map[string]interface{}, error) {
@@ -43,7 +43,7 @@ func getDynamicState(options plugininstaller.RawOptions) (map[string]interface{}
 
 	state := make(map[string]interface{})
 	operation := func() error {
-		err := util.GetArgoCDAppFromK8sAndSetState(state, opts.App.Name, opts.App.Namespace)
+		err := getArgoCDAppFromK8sAndSetState(state, opts.App.Name, opts.App.Namespace)
 		if err != nil {
 			return err
 		}
@@ -57,3 +57,22 @@ func getDynamicState(options plugininstaller.RawOptions) (map[string]interface{}
 	}
 	return state, nil
 }
+
+func getArgoCDAppFromK8sAndSetState(state map[string]interface{}, name, namespace string) error {
+	kubeClient, err := k8s.NewClient()
+	if err != nil {
+		return err
+	}
+
+	app, err := kubeClient.GetArgocdApplication(namespace, name)
+	if err != nil {
+		return err
+	}
+
+	d := kubeClient.DescribeArgocdApp(app)
+	state["app"] = d["app"]
+	state["src"] = d["src"]
+	state["dest"] = d["dest"]
+
+	return nil
+}

internal/pkg/plugin/devlake/create.go

@@ -11,7 +11,6 @@ func Create(options map[string]interface{}) (map[string]interface{}, error) {
 	runner := &plugininstaller.Runner{
 		ExecuteOperations: []plugininstaller.BaseOperation{
 			kubectl.ProcessByContent("create", devLakeInstallYAMLDownloadURL, ""),
-			kubectl.WaitDeployReadyWithDeployList(defaultNamespace, devLakeDeployments),
 		},
 		GetStatusOperation: getStaticState,
 	}

internal/pkg/plugin/devlake/state.go

@@ -2,7 +2,7 @@ package devlake
 
 import (
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller"
-	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/util"
+	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/common"
 )
 
 func getStaticState(opts plugininstaller.RawOptions) (map[string]interface{}, error) {
@@ -17,5 +17,8 @@ func getStaticState(opts plugininstaller.RawOptions) (map[string]interface{}, er
 }
 
 func getDynamicState(opts plugininstaller.RawOptions) (map[string]interface{}, error) {
-	return util.ReadDepAndServiceState(defaultNamespace, devLakeDeployments)
+	labelFilter := map[string]string{
+		"app": "devlake",
+	}
+	return common.GetPluginAllK8sState(defaultNamespace, map[string]string{}, labelFilter)
 }

internal/pkg/plugininstaller/reposcaffolding/dstrepo.go → internal/pkg/plugininstaller/common/repo.go

@@ -1,20 +1,21 @@
-package reposcaffolding
+package common
 
 import (
 	"io/fs"
 	"os"
 	"path/filepath"
+	"regexp"
 	"strings"
 
-	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/util"
+	"github.com/devstream-io/devstream/pkg/util/file"
 	"github.com/devstream-io/devstream/pkg/util/github"
 	"github.com/devstream-io/devstream/pkg/util/gitlab"
 	"github.com/devstream-io/devstream/pkg/util/log"
 	"github.com/devstream-io/devstream/pkg/util/template"
 )
 
-// DstRepo is the destination repo to push scaffolding project
-type DstRepo struct {
+// Repo is the repo info of github or gitlab
+type Repo struct {
 	Owner             string `validate:"required_without=Org"`
 	Org               string `validate:"required_without=Owner"`
 	Repo              string `validate:"required"`
@@ -26,17 +27,18 @@ type DstRepo struct {
 	Visibility string `validate:"omitempty,oneof=public private internal"`
 }
 
-func (d *DstRepo) createLocalRepoPath(workpath string) (string, error) {
+// CreateLocalRepoPath create local path for repo
+func (d *Repo) CreateLocalRepoPath(workpath string) (string, error) {
 	localPath := filepath.Join(workpath, d.Repo)
 	if err := os.MkdirAll(localPath, os.ModePerm); err != nil {
 		return "", err
 	}
 	return localPath, nil
 }
 
-// this method generate a walker func to render and copy files from srcRepoPath to dstRepoPath
-func (d *DstRepo) generateRenderWalker(
-	srcRepoPath, dstRepoPath string, renderConfig map[string]interface{},
+// Generate is a walker func to render and copy files from srcRepoPath to dstRepoPath
+func (d *Repo) GenerateRenderWalker(
+	srcRepoPath, dstRepoPath, appNamePlaceHolder string, renderConfig map[string]interface{},
 ) func(path string, info fs.FileInfo, err error) error {
 	return func(path string, info fs.FileInfo, err error) error {
 		if err != nil {
@@ -56,7 +58,7 @@ func (d *DstRepo) generateRenderWalker(
 		}
 
 		// replace template with appName
-		outputWithRepoName, err := replaceAppNameInPathStr(relativePath, d.Repo)
+		outputWithRepoName, err := replaceAppNameInPathStr(relativePath, appNamePlaceHolder, d.Repo)
 		if err != nil {
 			log.Debugf("Walk: Replace file name failed %s.", path)
 			return err
@@ -83,11 +85,12 @@ func (d *DstRepo) generateRenderWalker(
 			outputPath = strings.TrimSuffix(outputPath, ".tpl")
 			return template.RenderForFile("repo-scaffolding", path, outputPath, renderConfig)
 		}
-		return util.CopyFile(path, outputPath)
+		return file.CopyFile(path, outputPath)
 	}
 }
 
-func (d *DstRepo) createRepoRenderConfig() map[string]interface{} {
+// CreateRepoRenderConfig will generate template render variables
+func (d *Repo) CreateRepoRenderConfig() map[string]interface{} {
 	var owner = d.Owner
 	if d.Org != "" {
 		owner = d.Org
@@ -103,7 +106,8 @@ func (d *DstRepo) createRepoRenderConfig() map[string]interface{} {
 	return renderConfigMap
 }
 
-func (d *DstRepo) createGithubClient(needAuth bool) (*github.Client, error) {
+// CreateGithubClient build github client connection info
+func (d *Repo) CreateGithubClient(needAuth bool) (*github.Client, error) {
 	ghOptions := &github.Option{
 		Owner:    d.Owner,
 		Org:      d.Org,
@@ -117,15 +121,26 @@ func (d *DstRepo) createGithubClient(needAuth bool) (*github.Client, error) {
 	return ghClient, nil
 }
 
-func (d *DstRepo) createGitlabClient() (*gitlab.Client, error) {
+// CreateGitlabClient build gitlab connection info
+func (d *Repo) CreateGitlabClient() (*gitlab.Client, error) {
 	return gitlab.NewClient(gitlab.WithBaseURL(d.BaseURL))
 }
 
-func (d *DstRepo) buildgitlabOpts() *gitlab.CreateProjectOptions {
+// BuildgitlabOpts build gitlab connection options
+func (d *Repo) BuildgitlabOpts() *gitlab.CreateProjectOptions {
 	return &gitlab.CreateProjectOptions{
 		Name:       d.Repo,
 		Branch:     d.Branch,
 		Namespace:  d.Org,
 		Visibility: d.Visibility,
 	}
 }
+
+func replaceAppNameInPathStr(filePath, appNamePlaceHolder, appName string) (string, error) {
+	if !strings.Contains(filePath, appNamePlaceHolder) {
+		return filePath, nil
+	}
+	newFilePath := regexp.MustCompile(appNamePlaceHolder).ReplaceAllString(filePath, appName)
+	log.Debugf("Replace file path place holder. Before: %s, after: %s.", filePath, newFilePath)
+	return newFilePath, nil
+}

internal/pkg/plugininstaller/common/state.go

@@ -0,0 +1,39 @@
+package common
+
+import (
+	"github.com/devstream-io/devstream/pkg/util/helm"
+	"github.com/devstream-io/devstream/pkg/util/k8s"
+	"github.com/devstream-io/devstream/pkg/util/log"
+)
+
+func GetPluginAllK8sState(nameSpace string, anFilter, labelFilter map[string]string) (map[string]interface{}, error) {
+
+	// 1. init kube client
+	kubeClient, err := k8s.NewClient()
+	if err != nil {
+		return nil, err
+	}
+
+	// 2. get all related resource
+	allResource, err := kubeClient.GetResourceStatus(nameSpace, anFilter, labelFilter)
+	if err != nil {
+		log.Debugf("helm status: get status failed: %s", err)
+		return nil, err
+	}
+
+	// 3. transfer resources status to workflows
+	state := &helm.InstanceState{}
+	for _, dep := range allResource.Deployment {
+		state.Workflows.AddDeployment(dep.Name, dep.Ready)
+	}
+	for _, sts := range allResource.StatefulSet {
+		state.Workflows.AddStatefulset(sts.Name, sts.Ready)
+	}
+	for _, ds := range allResource.DaemonSet {
+		state.Workflows.AddDaemonset(ds.Name, ds.Ready)
+	}
+
+	retMap := state.ToStringInterfaceMap()
+	log.Debugf("Return map: %v.", retMap)
+	return retMap, nil
+}

internal/pkg/plugininstaller/helm/state.go

@@ -2,9 +2,8 @@ package helm
 
 import (
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller"
+	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/common"
 	"github.com/devstream-io/devstream/pkg/util/helm"
-	"github.com/devstream-io/devstream/pkg/util/k8s"
-	"github.com/devstream-io/devstream/pkg/util/log"
 )
 
 // GetPlugAllStateWrapper will get deploy, ds, statefulset status
@@ -14,35 +13,9 @@ func GetPluginAllState(options plugininstaller.RawOptions) (map[string]interface
 		return nil, err
 	}
 
-	// 1. init kube client
-	kubeClient, err := k8s.NewClient()
-	if err != nil {
-		return nil, err
-	}
 	anFilter := map[string]string{
 		helm.GetAnnotationName(): opts.GetReleaseName(),
 	}
-
-	// 2. get all related resource
-	allResource, err := kubeClient.GetResourceStatus(opts.GetNamespace(), anFilter)
-	if err != nil {
-		log.Debugf("helm status: get status failed: %s", err)
-		return nil, err
-	}
-
-	// 3. transfer resources status to workflows
-	state := &helm.InstanceState{}
-	for _, dep := range allResource.Deployment {
-		state.Workflows.AddDeployment(dep.Name, dep.Ready)
-	}
-	for _, sts := range allResource.StatefulSet {
-		state.Workflows.AddStatefulset(sts.Name, sts.Ready)
-	}
-	for _, ds := range allResource.DaemonSet {
-		state.Workflows.AddDaemonset(ds.Name, ds.Ready)
-	}
-
-	retMap := state.ToStringInterfaceMap()
-	log.Debugf("Return map: %v.", retMap)
-	return retMap, nil
+	labelFilter := map[string]string{}
+	return common.GetPluginAllK8sState(opts.GetNamespace(), anFilter, labelFilter)
 }

internal/pkg/plugininstaller/kubectl/installer.go

@@ -3,12 +3,8 @@ package kubectl
 import (
 	"fmt"
 	"os"
-	"time"
-
-	"github.com/cenkalti/backoff"
 
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller"
-	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/util"
 	"github.com/devstream-io/devstream/pkg/util/kubectl"
 	"github.com/devstream-io/devstream/pkg/util/log"
 )
@@ -43,23 +39,3 @@ func ProcessByContent(action, downloadUrl, content string) plugininstaller.BaseO
 		return nil
 	}
 }
-
-// WaitDeployReadyWithDeployList will wait all deploy in deployList get ready
-func WaitDeployReadyWithDeployList(namespace string, deployList []string) plugininstaller.BaseOperation {
-	waitFunc := func(options plugininstaller.RawOptions) error {
-		operation := func() error {
-			if err := util.CheckAllDeployAndServiceReady(namespace, deployList); err != nil {
-				return err
-			}
-			return nil
-		}
-		bkoff := backoff.NewExponentialBackOff()
-		bkoff.MaxElapsedTime = 3 * time.Minute
-		err := backoff.Retry(operation, bkoff)
-		if err != nil {
-			return err
-		}
-		return nil
-	}
-	return waitFunc
-}

internal/pkg/plugininstaller/reposcaffolding/installer.go

@@ -62,14 +62,14 @@ func DeleteRepo(options plugininstaller.RawOptions) error {
 	switch dstRepo.RepoType {
 	case "github":
 		// 1. create ghClient
-		ghClient, err := opts.DestinationRepo.createGithubClient(true)
+		ghClient, err := opts.DestinationRepo.CreateGithubClient(true)
 		if err != nil {
 			return err
 		}
 		// 2. delete github repo
 		return ghClient.DeleteRepo()
 	case "gitlab":
-		gLclient, err := dstRepo.createGitlabClient()
+		gLclient, err := dstRepo.CreateGitlabClient()
 		if err != nil {
 			return err
 		}

internal/pkg/plugininstaller/reposcaffolding/option.go

@@ -4,18 +4,19 @@ import (
 	"github.com/mitchellh/mapstructure"
 
 	"github.com/devstream-io/devstream/internal/pkg/plugininstaller"
+	"github.com/devstream-io/devstream/internal/pkg/plugininstaller/common"
 	"github.com/devstream-io/devstream/pkg/util/log"
 )
 
 const (
 	transitBranch      = "init-with-devstream"
-	appNamePlaceHolder = "_app_name_"
 	defaultCommitMsg   = "init with devstream"
+	appNamePlaceHolder = "_app_name_"
 )
 
 type Options struct {
-	SourceRepo      *SrcRepo `validate:"required" mapstructure:"source_repo"`
-	DestinationRepo *DstRepo `validate:"required" mapstructure:"destination_repo"`
+	SourceRepo      *SrcRepo     `validate:"required" mapstructure:"source_repo"`
+	DestinationRepo *common.Repo `validate:"required" mapstructure:"destination_repo"`
 	Vars            map[string]interface{}
 }
 
@@ -53,14 +54,14 @@ func (opts *Options) CreateAndRenderLocalRepo(workpath string) error {
 func (opts *Options) PushToRemoteGitlab(repoPath string) error {
 	dstRepo := opts.DestinationRepo
 	// 1. init gitlab client
-	c, err := dstRepo.createGitlabClient()
+	c, err := dstRepo.CreateGitlabClient()
 	if err != nil {
 		log.Debugf("Gitlab push: init gitlab client failed %s", err)
 		return err
 	}
 
 	// 2. create the project
-	if err := c.CreateProject(dstRepo.buildgitlabOpts()); err != nil {
+	if err := c.CreateProject(dstRepo.BuildgitlabOpts()); err != nil {
 		log.Errorf("Failed to create repo: %s.", err)
 		return err
 	}
@@ -91,7 +92,7 @@ func (opts *Options) PushToRemoteGitlab(repoPath string) error {
 func (opts *Options) PushToRemoteGithub(repoPath string) error {
 	dstRepo := opts.DestinationRepo
 	// 1. init github client
-	ghClient, err := dstRepo.createGithubClient(true)
+	ghClient, err := dstRepo.CreateGithubClient(true)
 	if err != nil {
 		log.Debugf("Github push: init github client failed %s", err)
 		return err
@@ -123,7 +124,7 @@ func (opts *Options) PushToRemoteGithub(repoPath string) error {
 }
 
 func (opts *Options) renderTplConfig() map[string]interface{} {
-	renderConfig := opts.DestinationRepo.createRepoRenderConfig()
+	renderConfig := opts.DestinationRepo.CreateRepoRenderConfig()
 	for k, v := range opts.Vars {
 		renderConfig[k] = v
 	}