feat: add the unified metadata manager (#158)

* refactor: use 'EnsureDir()'

Signed-off-by: zyy17 <zyylsxm@gmail.com>

* refactore: use new artifacts manager in bare-metal deployment

Signed-off-by: zyy17 <zyylsxm@gmail.com>

* feat: add the metadata manager

* refactor: use metadata manager in bare-metal deployment

* refactor: add '--enable-cache' to replace '--always-download'

* tests: add unit tests of metadata manager

---------

Signed-off-by: zyy17 <zyylsxm@gmail.com>

pkg/artifacts/manager.go

@@ -82,8 +82,11 @@ type Source struct {
 
 // DownloadOptions is the options for downloading the artifact.
 type DownloadOptions struct {
-	// If UseCache is true, the manager will use the cache if the artifact already exists.
-	UseCache bool
+	// If EnableCache is true, the manager will use the cache if the artifact already exists.
+	EnableCache bool
+
+	// If the artifact is a binary, the manager will install the binary to the BinaryInstallDir after downloading its package.
+	BinaryInstallDir string
 }
 
 // manager is the implementation of Manager interface.
@@ -185,7 +188,7 @@ func (m *manager) NewSource(name, version string, typ ArtifactType, fromCNRegion
 func (m *manager) DownloadTo(ctx context.Context, from *Source, destDir string, opts *DownloadOptions) (string, error) {
 	artifactFile := filepath.Join(destDir, from.FileName)
 	shouldDownload := true
-	if opts.UseCache {
+	if opts.EnableCache {
 		_, err := os.Stat(artifactFile)
 
 		// If the file exists, skip downloading.
@@ -222,8 +225,10 @@ func (m *manager) DownloadTo(ctx context.Context, from *Source, destDir string,
 	}
 
 	if from.Type == ArtifactTypeBinary {
-		installDir := filepath.Join(filepath.Dir(destDir), "bin")
-		if err := m.installBinaries(artifactFile, installDir); err != nil {
+		if opts.BinaryInstallDir == "" {
+			return "", fmt.Errorf("binary install dir is empty")
+		}
+		if err := m.installBinaries(artifactFile, opts.BinaryInstallDir); err != nil {
 			return "", err
 		}
 		return filepath.Join(filepath.Dir(destDir), "bin", from.Name), nil

pkg/artifacts/manager_test.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"fmt"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"sigs.k8s.io/kind/pkg/log"
@@ -56,7 +57,7 @@ func TestDownloadCharts(t *testing.T) {
 		if err != nil {
 			t.Errorf("failed to create source: %v", err)
 		}
-		artifactFile, err := m.DownloadTo(ctx, src, destDir(tempDir, src), &DownloadOptions{UseCache: false})
+		artifactFile, err := m.DownloadTo(ctx, src, destDir(tempDir, src), &DownloadOptions{EnableCache: false})
 		if err != nil {
 			t.Errorf("failed to download: %v", err)
 		}
@@ -102,7 +103,7 @@ func TestDownloadChartsFromCNRegion(t *testing.T) {
 		if err != nil {
 			t.Errorf("failed to create source: %v", err)
 		}
-		artifactFile, err := m.DownloadTo(ctx, src, destDir(tempDir, src), &DownloadOptions{UseCache: false})
+		artifactFile, err := m.DownloadTo(ctx, src, destDir(tempDir, src), &DownloadOptions{EnableCache: false})
 		if err != nil {
 			t.Errorf("failed to download: %v", err)
 		}
@@ -192,7 +193,7 @@ func TestDownloadBinaries(t *testing.T) {
 		if err != nil {
 			t.Errorf("failed to create source: %v", err)
 		}
-		artifactFile, err := m.DownloadTo(ctx, src, destDir(tempDir, src), &DownloadOptions{UseCache: false})
+		artifactFile, err := m.DownloadTo(ctx, src, destDir(tempDir, src), &DownloadOptions{EnableCache: false, BinaryInstallDir: filepath.Join(filepath.Dir(destDir(tempDir, src)), "bin")})
 		if err != nil {
 			t.Errorf("failed to download: %v", err)
 		}
@@ -228,7 +229,7 @@ func TestArtifactsCache(t *testing.T) {
 	if err != nil {
 		t.Errorf("failed to create source: %v", err)
 	}
-	artifactFile, err := m.DownloadTo(ctx, src, destDir(tempDir, src), &DownloadOptions{UseCache: false})
+	artifactFile, err := m.DownloadTo(ctx, src, destDir(tempDir, src), &DownloadOptions{EnableCache: false})
 	if err != nil {
 		t.Errorf("failed to download: %v", err)
 	}
@@ -242,7 +243,7 @@ func TestArtifactsCache(t *testing.T) {
 	}
 
 	// Download again with cache.
-	artifactFile, err = m.DownloadTo(ctx, src, destDir(tempDir, src), &DownloadOptions{UseCache: true})
+	artifactFile, err = m.DownloadTo(ctx, src, destDir(tempDir, src), &DownloadOptions{EnableCache: true})
 	if err != nil {
 		t.Errorf("failed to download: %v", err)
 	}

pkg/cmd/gtctl/cluster/create/create.go

@@ -57,7 +57,7 @@ type ClusterCliOptions struct {
 	BareMetal          bool
 	Config             string
 	GreptimeBinVersion string
-	AlwaysDownload     bool
+	EnableCache        bool
 	RetainLogs         bool
 
 	// Common options.
@@ -104,7 +104,7 @@ func NewCreateClusterCommand(l logger.Logger) *cobra.Command {
 	cmd.Flags().BoolVar(&options.BareMetal, "bare-metal", false, "Deploy the greptimedb cluster on bare-metal environment.")
 	cmd.Flags().StringVar(&options.GreptimeBinVersion, "greptime-bin-version", "", "The version of greptime binary(can be override by config file).")
 	cmd.Flags().StringVar(&options.Config, "config", "", "Configuration to deploy the greptimedb cluster on bare-metal environment.")
-	cmd.Flags().BoolVar(&options.AlwaysDownload, "always-download", false, "If true, always download the binary.")
+	cmd.Flags().BoolVar(&options.EnableCache, "enable-cache", true, "If true, enable cache for downloading artifacts(charts and binaries).")
 	cmd.Flags().BoolVar(&options.RetainLogs, "retain-logs", true, "If true, always retain the logs of binary.")
 	cmd.Flags().BoolVar(&options.UseGreptimeCNArtifacts, "use-greptime-cn-artifacts", false, "If true, use greptime-cn artifacts(charts and binaries).")
 
@@ -218,7 +218,7 @@ func newDeployer(l logger.Logger, clusterName string, options *ClusterCliOptions
 		opts = append(opts, baremetal.WithMergeConfig(&cfg, raw))
 	}
 
-	opts = append(opts, baremetal.WithAlawaysDownload(options.AlwaysDownload))
+	opts = append(opts, baremetal.WithEnableCache(options.EnableCache))
 
 	baremetalDeployer, err := baremetal.NewDeployer(l, clusterName, opts...)
 	if err != nil {

pkg/cmd/gtctl/playground/playground.go

@@ -35,10 +35,10 @@ func NewPlaygroundCommand(l logger.Logger) *cobra.Command {
 
 			playgroundName := codename.Generate(rng, 0)
 			playgroundOptions := create.ClusterCliOptions{
-				BareMetal:      true,
-				RetainLogs:     false,
-				Timeout:        900, // 15min
-				AlwaysDownload: false,
+				BareMetal:   true,
+				RetainLogs:  false,
+				Timeout:     900, // 15min
+				EnableCache: false,
 			}
 
 			if err := create.NewCluster([]string{playgroundName}, playgroundOptions, l); err != nil {

pkg/deployer/baremetal/deployer.go

@@ -21,7 +21,6 @@ import (
 	"os/exec"
 	"os/signal"
 	"path"
-	"path/filepath"
 	"strings"
 	"sync"
 	"syscall"
@@ -34,13 +33,13 @@ import (
 	"github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/component"
 	"github.com/GreptimeTeam/gtctl/pkg/deployer/baremetal/config"
 	"github.com/GreptimeTeam/gtctl/pkg/logger"
+	"github.com/GreptimeTeam/gtctl/pkg/metadata"
 	fileutils "github.com/GreptimeTeam/gtctl/pkg/utils/file"
 )
 
 type Deployer struct {
 	logger logger.Logger
 	config *config.Config
-	am     artifacts.Manager
 	wg     sync.WaitGroup
 	bm     *component.BareMetalCluster
 	ctx    context.Context
@@ -51,7 +50,10 @@ type Deployer struct {
 	baseDir           string
 	clusterConfigPath string
 
-	alwaysDownload bool
+	am artifacts.Manager
+	mm metadata.Manager
+
+	enableCache bool
 }
 
 var _ Interface = &Deployer{}
@@ -67,6 +69,12 @@ func NewDeployer(l logger.Logger, clusterName string, opts ...Option) (Interface
 		ctx:    ctx,
 	}
 
+	mm, err := metadata.New("")
+	if err != nil {
+		return nil, err
+	}
+	d.mm = mm
+
 	for _, opt := range opts {
 		if opt != nil {
 			opt(d)
@@ -233,9 +241,9 @@ func WithGreptimeVersion(version string) Option {
 	}
 }
 
-func WithAlawaysDownload(alwaysDownload bool) Option {
+func WithEnableCache(enableCache bool) Option {
 	return func(d *Deployer) {
-		d.alwaysDownload = alwaysDownload
+		d.enableCache = enableCache
 	}
 }
 
@@ -297,8 +305,17 @@ func (d *Deployer) CreateGreptimeDBCluster(ctx context.Context, clusterName stri
 				return err
 			}
 
-			destDir := filepath.Join(d.baseDir, "artifacts", "binaries", artifacts.GreptimeBinName, d.config.Cluster.Artifact.Version, "pkg")
-			artifactFile, err := d.am.DownloadTo(ctx, src, destDir, &artifacts.DownloadOptions{UseCache: true})
+			destDir, err := d.mm.AllocateArtifactFilePath(src, false)
+			if err != nil {
+				return err
+			}
+
+			installDir, err := d.mm.AllocateArtifactFilePath(src, true)
+			if err != nil {
+				return err
+			}
+
+			artifactFile, err := d.am.DownloadTo(ctx, src, destDir, &artifacts.DownloadOptions{EnableCache: d.enableCache, BinaryInstallDir: installDir})
 			if err != nil {
 				return err
 			}
@@ -367,8 +384,17 @@ func (d *Deployer) CreateEtcdCluster(ctx context.Context, clusterName string, op
 				return err
 			}
 
-			destDir := filepath.Join(d.baseDir, "artifacts", "binaries", artifacts.EtcdBinName, d.config.Etcd.Artifact.Version, "pkg")
-			artifactFile, err := d.am.DownloadTo(ctx, src, destDir, &artifacts.DownloadOptions{UseCache: true})
+			destDir, err := d.mm.AllocateArtifactFilePath(src, false)
+			if err != nil {
+				return err
+			}
+
+			installDir, err := d.mm.AllocateArtifactFilePath(src, true)
+			if err != nil {
+				return err
+			}
+
+			artifactFile, err := d.am.DownloadTo(ctx, src, destDir, &artifacts.DownloadOptions{EnableCache: d.enableCache, BinaryInstallDir: installDir})
 			if err != nil {
 				return err
 			}

pkg/deployer/baremetal/deployer_test.go

@@ -51,15 +51,15 @@ func TestNewDeployer(t *testing.T) {
 		LogsDir: path.Join(homedir, clusterName, "logs"),
 		PidsDir: path.Join(homedir, clusterName, "pids"),
 	})
-	assert.False(t, d1.alwaysDownload)
+	assert.False(t, d1.enableCache)
 
 	// New Deployer with always download option
-	deployer, err = NewDeployer(L, clusterName, WithAlawaysDownload(true))
+	deployer, err = NewDeployer(L, clusterName, WithEnableCache(true))
 	assert.NoError(t, err)
 	d2, ok := deployer.(*Deployer)
 	assert.True(t, ok)
 	assert.NotNil(t, d2)
-	assert.True(t, d2.alwaysDownload)
+	assert.True(t, d2.enableCache)
 
 	// New Deployer with replace config option
 	newConfig := config.DefaultConfig()

pkg/helm/helm.go

@@ -19,7 +19,6 @@ import (
 	"context"
 	"fmt"
 	"os"
-	"path/filepath"
 	"reflect"
 	"strings"
 
@@ -31,6 +30,7 @@ import (
 
 	"github.com/GreptimeTeam/gtctl/pkg/artifacts"
 	"github.com/GreptimeTeam/gtctl/pkg/logger"
+	"github.com/GreptimeTeam/gtctl/pkg/metadata"
 )
 
 const (
@@ -39,7 +39,7 @@ const (
 
 var (
 	// KubeVersion is the target version of the kubernetes.
-	KubeVersion string = "v1.20.0"
+	KubeVersion = "v1.20.0"
 )
 
 // Manager is the Helm charts manager. The implementation is based on Helm SDK.
@@ -50,11 +50,11 @@ type Manager struct {
 	// logger is the logger for the Manager.
 	logger logger.Logger
 
-	// artifactsManager is the artifacts manager to manage charts.
+	// am is the artifacts manager to manage charts.
 	am artifacts.Manager
 
-	// metadataDir is the directory to store the metadata of the gtctl.
-	metadataDir string
+	// mm is the metadata manager to manage the metadata.
+	mm metadata.Manager
 }
 
 type Option func(*Manager)
@@ -68,12 +68,11 @@ func NewManager(l logger.Logger, opts ...Option) (*Manager, error) {
 	}
 	r.am = am
 
-	// TODO(zyy17): The metadataDir will be managed by the independent manager in the future.
-	homeDir, err := os.UserHomeDir()
+	mm, err := metadata.New("")
 	if err != nil {
 		return nil, err
 	}
-	r.metadataDir = filepath.Join(homeDir, ".gtctl")
+	r.mm = mm
 
 	for _, opt := range opts {
 		opt(r)
@@ -82,9 +81,14 @@ func NewManager(l logger.Logger, opts ...Option) (*Manager, error) {
 	return r, nil
 }
 
-func WithMetadataDir(dir string) Option {
+func WithHomeDir(dir string) Option {
 	return func(r *Manager) {
-		r.metadataDir = dir
+		metadataManager, err := metadata.New(dir)
+		if err != nil {
+			r.logger.Errorf("failed to create metadata manager: %v", err)
+			os.Exit(1)
+		}
+		r.mm = metadataManager
 	}
 }
 
@@ -105,8 +109,12 @@ func (r *Manager) LoadAndRenderChart(ctx context.Context, name, namespace, chart
 		return nil, err
 	}
 
-	destDir := filepath.Join(r.metadataDir, "artifacts", "charts", chartName, chartVersion)
-	chartFile, err := r.am.DownloadTo(ctx, src, destDir, &artifacts.DownloadOptions{UseCache: true})
+	destDir, err := r.mm.AllocateArtifactFilePath(src, false)
+	if err != nil {
+		return nil, err
+	}
+
+	chartFile, err := r.am.DownloadTo(ctx, src, destDir, &artifacts.DownloadOptions{EnableCache: true})
 	if err != nil {
 		return nil, err
 	}

pkg/helm/helm_test.go

@@ -34,7 +34,7 @@ const (
 )
 
 func TestLoadAndRenderChart(t *testing.T) {
-	r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), WithMetadataDir(testMetadataDir))
+	r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), WithHomeDir(testMetadataDir))
 	if err != nil {
 		t.Errorf("failed to create render: %v", err)
 	}
@@ -85,7 +85,7 @@ func TestLoadAndRenderChart(t *testing.T) {
 }
 
 func TestRender_GenerateGreptimeDBHelmValues(t *testing.T) {
-	r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), WithMetadataDir(testMetadataDir))
+	r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), WithHomeDir(testMetadataDir))
 	if err != nil {
 		t.Errorf("failed to create render: %v", err)
 	}
@@ -129,7 +129,7 @@ func TestRender_GenerateGreptimeDBHelmValues(t *testing.T) {
 }
 
 func TestRender_GenerateGreptimeDBOperatorHelmValues(t *testing.T) {
-	r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), WithMetadataDir(testMetadataDir))
+	r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), WithHomeDir(testMetadataDir))
 	if err != nil {
 		t.Errorf("failed to create render: %v", err)
 	}
@@ -163,7 +163,7 @@ func TestRender_GenerateGreptimeDBOperatorHelmValues(t *testing.T) {
 }
 
 func TestRender_GenerateEtcdHelmValues(t *testing.T) {
-	r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), WithMetadataDir(testMetadataDir))
+	r, err := NewManager(logger.New(os.Stdout, log.Level(4), logger.WithColored()), WithHomeDir(testMetadataDir))
 	if err != nil {
 		t.Errorf("failed to create render: %v", err)
 	}