Add Support for ES v8.2.0 dashboard backup and restore (#1459)

* Add elastic dashboard backup and restore support

Signed-off-by: Md. Ishtiaq Islam <ishtiaq@appscode.com>

* Add os dashboard backup and restore support

Signed-off-by: Md. Ishtiaq Islam <ishtiaq@appscode.com>

---------

Signed-off-by: Md. Ishtiaq Islam <ishtiaq@appscode.com>

Author: ishtiaqhimel

go.mod

@@ -13,27 +13,36 @@ require (
 	k8s.io/apimachinery v0.29.0
 	k8s.io/client-go v0.29.0
 	k8s.io/klog/v2 v2.110.1
-	kmodules.xyz/client-go v0.29.4
+	kmodules.xyz/client-go v0.29.6
 	kmodules.xyz/custom-resources v0.29.0
 	kmodules.xyz/offshoot-api v0.29.0
-	stash.appscode.dev/apimachinery v0.32.1-0.20240101013736-ef308633d8b2
+	kubedb.dev/apimachinery v0.41.0-beta.1.0.20240124061503-ce4799bb0e5c
+	kubedb.dev/db-client-go v0.0.9-0.20240126103627-22edae9f6b92
+	sigs.k8s.io/controller-runtime v0.16.3
+	stash.appscode.dev/apimachinery v0.32.1-0.20240118085630-4c06ed8c04a7
 )
 
 require (
 	github.com/Masterminds/semver/v3 v3.2.1 // indirect
 	github.com/PuerkitoBio/purell v1.2.0 // indirect
 	github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect
+	github.com/beorn7/perks v1.0.1 // indirect
+	github.com/cert-manager/cert-manager v1.13.3 // indirect
+	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
 	github.com/evanphx/json-patch v5.7.0+incompatible // indirect
 	github.com/evanphx/json-patch/v5 v5.7.0 // indirect
 	github.com/fatih/structs v1.1.0 // indirect
-	github.com/go-logr/logr v1.3.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/go-logr/logr v1.4.1 // indirect
 	github.com/go-openapi/jsonpointer v0.20.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
 	github.com/go-openapi/swag v0.22.4 // indirect
+	github.com/go-resty/resty/v2 v2.11.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/google/gnostic-models v0.6.8 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
@@ -44,15 +53,18 @@ require (
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
-	github.com/mattn/go-isatty v0.0.18 // indirect
+	github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/onsi/gomega v1.30.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
-	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/rogpeppe/go-internal v1.11.0 // indirect
+	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.70.0 // indirect
+	github.com/prometheus/client_golang v1.18.0 // indirect
+	github.com/prometheus/client_model v0.5.0 // indirect
+	github.com/prometheus/common v0.45.0 // indirect
+	github.com/prometheus/procfs v0.12.0 // indirect
 	github.com/sergi/go-diff v1.2.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/yudai/gojsondiff v1.0.0 // indirect
@@ -79,13 +91,15 @@ require (
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiextensions-apiserver v0.29.0 // indirect
 	k8s.io/apiserver v0.29.0 // indirect
+	k8s.io/component-base v0.29.0 // indirect
 	k8s.io/kube-aggregator v0.29.0 // indirect
 	k8s.io/kube-openapi v0.0.0-20231129212854-f0671cc7e66a // indirect
 	k8s.io/utils v0.0.0-20231127182322-b307cd553661 // indirect
 	kmodules.xyz/apiversion v0.2.0 // indirect
+	kmodules.xyz/monitoring-agent-api v0.29.0 // indirect
 	kmodules.xyz/objectstore-api v0.29.0 // indirect
 	kmodules.xyz/prober v0.29.0 // indirect
-	sigs.k8s.io/controller-runtime v0.16.3 // indirect
+	sigs.k8s.io/gateway-api v0.8.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
 	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
 	sigs.k8s.io/yaml v1.4.0 // indirect

go.sum

@@ -19,6 +19,8 @@ package pkg
 import (
 	"context"
 	"fmt"
+	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 	"time"
@@ -140,6 +142,7 @@ func NewCmdBackup() *cobra.Command {
 
 	cmd.Flags().StringVar(&opt.backupOptions.Host, "hostname", opt.backupOptions.Host, "Name of the host machine")
 	cmd.Flags().StringVar(&opt.interimDataDir, "interim-data-dir", opt.interimDataDir, "Directory where the targeted data will be stored temporarily before uploading to the backend.")
+	cmd.Flags().BoolVar(&opt.enableDashboard, "enable-dashboard-backup", opt.enableDashboard, "Specify whether to enable kibana dashboard backup")
 
 	cmd.Flags().Int64Var(&opt.backupOptions.RetentionPolicy.KeepLast, "retention-keep-last", opt.backupOptions.RetentionPolicy.KeepLast, "Specify value for retention strategy")
 	cmd.Flags().Int64Var(&opt.backupOptions.RetentionPolicy.KeepHourly, "retention-keep-hourly", opt.backupOptions.RetentionPolicy.KeepHourly, "Specify value for retention strategy")
@@ -242,6 +245,11 @@ func (opt *esOptions) backupElasticsearch(targetRef api_v1beta1.TargetRef) (*res
 	if err := opt.dumpPVCStorageLimit(targetRef); err != nil {
 		return nil, fmt.Errorf("failed to dump pvc storage limit info %w", err)
 	}
+
+	if err = opt.dumpDashboardObjects(appBinding); err != nil {
+		return nil, fmt.Errorf("failed to dump dashboard objects %w", err)
+	}
+
 	// dumped data has been stored in the interim data dir. Now, we will backup this directory using Stash.
 	opt.backupOptions.BackupPaths = []string{opt.interimDataDir}
 
@@ -288,3 +296,30 @@ func (opt *esOptions) dumpPVCStorageLimit(targetRef api_v1beta1.TargetRef) error
 func targetMatched(tref api_v1beta1.TargetRef, expectedKind, expectedName, expectedNamespace string) bool {
 	return tref.Kind == expectedKind && tref.Namespace == expectedNamespace && tref.Name == expectedName
 }
+
+func (opt *esOptions) dumpDashboardObjects(appBinding *appcatalog.AppBinding) error {
+	if !opt.enableDashboard {
+		return nil
+	}
+
+	dashboardClient, err := opt.getDashboardClient(appBinding)
+	if err != nil {
+		return err
+	}
+
+	response, err := dashboardClient.ExportSavedObjects()
+	if err != nil {
+		return err
+	}
+
+	body, err := io.ReadAll(response.Body)
+	if err != nil {
+		return err
+	}
+
+	if response.Code != http.StatusOK {
+		return fmt.Errorf("failed to export dashboard saved objects %s", string(body))
+	}
+
+	return os.WriteFile(filepath.Join(opt.interimDataDir, DashboardObjectsFile), body, os.ModePerm)
+}

pkg/backup.go

@@ -19,6 +19,8 @@ package pkg
 import (
 	"context"
 	"fmt"
+	"io"
+	"net/http"
 	"os"
 	"path/filepath"
 
@@ -131,6 +133,7 @@ func NewCmdRestore() *cobra.Command {
 	cmd.Flags().StringVar(&opt.restoreOptions.SourceHost, "source-hostname", opt.restoreOptions.SourceHost, "Name of the host whose data will be restored")
 	cmd.Flags().StringSliceVar(&opt.restoreOptions.Snapshots, "snapshot", opt.restoreOptions.Snapshots, "Snapshots to restore")
 	cmd.Flags().StringVar(&opt.interimDataDir, "interim-data-dir", opt.interimDataDir, "Directory where the restored data will be stored temporarily before injecting into the desired database.")
+	cmd.Flags().BoolVar(&opt.enableDashboard, "enable-dashboard-restore", opt.enableDashboard, "Specify whether to enable kibana dashboard restore")
 
 	cmd.Flags().StringVar(&opt.outputDir, "output-dir", opt.outputDir, "Directory where output.json file will be written (keep empty if you don't need to write output in file)")
 
@@ -218,6 +221,15 @@ func (opt *esOptions) restoreElasticsearch(targetRef api_v1beta1.TargetRef) (*re
 		return nil, err
 	}
 
+	if err = opt.restoreDashboardObjects(appBinding); err != nil {
+		return nil, fmt.Errorf("failed to restore dashboard objects %w", err)
+	}
+
+	// delete the metadata file as it is not required for restoring the dumps
+	if err := clearFile(filepath.Join(opt.interimDataDir, DashboardObjectsFile)); err != nil {
+		return nil, err
+	}
+
 	// run separate shell to restore indices
 	// klog.Infoln("Performing multielasticdump on", hostname)
 	session.sh.ShowCMD = false
@@ -238,3 +250,30 @@ func clearFile(filepath string) error {
 	}
 	return nil
 }
+
+func (opt *esOptions) restoreDashboardObjects(appBinding *appcatalog.AppBinding) error {
+	if !opt.enableDashboard {
+		return nil
+	}
+
+	dashboardClient, err := opt.getDashboardClient(appBinding)
+	if err != nil {
+		return err
+	}
+
+	response, err := dashboardClient.ImportSavedObjects(filepath.Join(opt.interimDataDir, DashboardObjectsFile))
+	if err != nil {
+		return err
+	}
+
+	body, err := io.ReadAll(response.Body)
+	if err != nil {
+		return err
+	}
+
+	if response.Code != http.StatusOK {
+		return fmt.Errorf("failed to import dashboard saved objects %s", string(body))
+	}
+
+	return nil
+}

pkg/restore.go

@@ -31,21 +31,31 @@ import (
 	shell "gomodules.xyz/go-sh"
 	core "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
 	kmapi "kmodules.xyz/client-go/api/v1"
 	meta_util "kmodules.xyz/client-go/meta"
 	appcatalog "kmodules.xyz/custom-resources/apis/appcatalog/v1alpha1"
 	appcatalog_cs "kmodules.xyz/custom-resources/client/clientset/versioned"
+	catalog "kubedb.dev/apimachinery/apis/catalog/v1alpha1"
+	esapi "kubedb.dev/apimachinery/apis/elasticsearch/v1alpha1"
+	kubedbapi "kubedb.dev/apimachinery/apis/kubedb/v1alpha2"
+	es_dashboard "kubedb.dev/db-client-go/elasticsearchdashboard"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
 )
 
 const (
-	ESUser              = "ADMIN_USERNAME"
-	ESPassword          = "ADMIN_PASSWORD"
-	MultiElasticDumpCMD = "multielasticdump"
-	ESCACertFile        = "root.pem"
-	ESAuthFile          = "auth.txt"
+	ESUser               = "ADMIN_USERNAME"
+	ESPassword           = "ADMIN_PASSWORD"
+	MultiElasticDumpCMD  = "multielasticdump"
+	ESCACertFile         = "root.pem"
+	ESAuthFile           = "auth.txt"
+	DashboardObjectsFile = "dashboard.ndjson"
 )
 
 type esOptions struct {
@@ -59,6 +69,7 @@ type esOptions struct {
 	appBindingNamespace string
 	esArgs              string
 	interimDataDir      string
+	enableDashboard     bool
 	outputDir           string
 	storageSecret       kmapi.ObjectReference
 	waitTimeout         int32
@@ -166,3 +177,119 @@ func writeAuthFile(filename string, cred *core.Secret) error {
 	)
 	return os.WriteFile(filename, []byte(authKeys), 0o400) // only readable to owner
 }
+
+func newRuntimeClient(cfg *rest.Config) (client.Client, error) {
+	scheme := runtime.NewScheme()
+	utilruntime.Must(core.AddToScheme(scheme))
+	utilruntime.Must(esapi.AddToScheme(scheme))
+	utilruntime.Must(kubedbapi.AddToScheme(scheme))
+
+	hc, err := rest.HTTPClientFor(cfg)
+	if err != nil {
+		return nil, err
+	}
+	mapper, err := apiutil.NewDynamicRESTMapper(cfg, hc)
+	if err != nil {
+		return nil, err
+	}
+
+	return client.New(cfg, client.Options{
+		Scheme: scheme,
+		Mapper: mapper,
+	})
+}
+
+func getElasticSearchDashboard(klient client.Client, appBinding *appcatalog.AppBinding) (*esapi.ElasticsearchDashboard, error) {
+	dashboards := &esapi.ElasticsearchDashboardList{}
+	opts := []client.ListOption{client.InNamespace(appBinding.Namespace)}
+	if err := klient.List(context.TODO(), dashboards, opts...); err != nil {
+		return nil, err
+	}
+
+	for _, dashboard := range dashboards.Items {
+		if dashboard.Spec.DatabaseRef != nil {
+			if dashboard.Spec.DatabaseRef.Name == appBinding.Name {
+				return &dashboard, nil
+			}
+		}
+	}
+
+	return nil, fmt.Errorf("no elasticsearch dashboard found")
+}
+
+func getElasticSearch(klient client.Client, appBinding *appcatalog.AppBinding) (*kubedbapi.Elasticsearch, error) {
+	es := &kubedbapi.Elasticsearch{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      appBinding.Name,
+			Namespace: appBinding.Namespace,
+		},
+	}
+
+	if err := klient.Get(context.TODO(), client.ObjectKeyFromObject(es), es); err != nil {
+		return nil, err
+	}
+
+	return es, nil
+}
+
+func getSecret(klient client.Client, obj kmapi.ObjectReference) (*core.Secret, error) {
+	sec := &core.Secret{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      obj.Name,
+			Namespace: obj.Namespace,
+		},
+	}
+
+	if err := klient.Get(context.TODO(), client.ObjectKeyFromObject(sec), sec); err != nil {
+		return nil, err
+	}
+
+	return sec, nil
+}
+
+func getVersionInfo(es *kubedbapi.Elasticsearch, appBinding *appcatalog.AppBinding) *es_dashboard.DbVersionInfo {
+	authPlugin := catalog.ElasticsearchAuthPluginOpenSearch
+	segments := strings.Split(es.Spec.Version, "-")
+	if segments[0] == "xpack" {
+		authPlugin = catalog.ElasticsearchAuthPluginXpack
+	}
+	return &es_dashboard.DbVersionInfo{
+		Name:       es.Spec.Version,
+		Version:    appBinding.Spec.Version,
+		AuthPlugin: authPlugin,
+	}
+}
+
+func (opt esOptions) getDashboardClient(appBinding *appcatalog.AppBinding) (*es_dashboard.Client, error) {
+	klient, err := newRuntimeClient(opt.config)
+	if err != nil {
+		return nil, err
+	}
+
+	esDashboard, err := getElasticSearchDashboard(klient, appBinding)
+	if err != nil {
+		return nil, err
+	}
+
+	es, err := getElasticSearch(klient, appBinding)
+	if err != nil {
+		return nil, err
+	}
+
+	sec, err := getSecret(klient, kmapi.ObjectReference{
+		Name:      es.Spec.AuthSecret.Name,
+		Namespace: es.Namespace,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	versionInfo := getVersionInfo(es, appBinding)
+
+	return es_dashboard.NewKubeDBClientBuilder(klient, esDashboard).
+		WithContext(context.TODO()).
+		WithDatabaseRef(es).
+		WithAuthSecret(sec).
+		WithDbVersionInfo(versionInfo).
+		GetElasticsearchDashboardClient()
+}