Added compactor subcommand.

.gitignore

@@ -22,6 +22,7 @@ test/output
 test/e2e_test_data
 default.bkp*
 default.etcd*
+compctedsnap.bkp*
 
 # IDE config
 .vscode

cmd/compact.go

@@ -0,0 +1,100 @@
+// Copyright (c) 2018 SAP SE or an SAP affiliate company. All rights reserved. This file is licensed under the Apache Software License, v. 2 except as noted otherwise in the LICENSE file.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"context"
+	"fmt"
+	"path/filepath"
+
+	"github.com/gardener/etcd-backup-restore/pkg/compactor"
+	"github.com/gardener/etcd-backup-restore/pkg/miscellaneous"
+	"github.com/gardener/etcd-backup-restore/pkg/snapstore"
+	brtypes "github.com/gardener/etcd-backup-restore/pkg/types"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"go.etcd.io/etcd/pkg/types"
+)
+
+// NewCompactCommand compacts the ETCD instance
+func NewCompactCommand(ctx context.Context) *cobra.Command {
+	opts := newCompactOptions()
+	// compactCmd represents the restore command
+	compactCmd := &cobra.Command{
+		Use:   "compact",
+		Short: "compacts multiple incremental snapshots in etcd backup into a single full snapshot",
+		Long:  fmt.Sprintf(`Compacts an existing backup stored in snapshot store.`),
+		Run: func(cmd *cobra.Command, args []string) {
+			/* Compact operation
+			- Restore from all the latest snapshots (Base + Delta).
+			- Compact the newly created embedded ETCD instance.
+			- Defragment
+			- Save the snapshot
+			*/
+			logger := logrus.New()
+			if err := opts.validate(); err != nil {
+				logger.Fatalf("failed to validate the options: %v", err)
+				return
+			}
+
+			opts.complete()
+
+			clusterUrlsMap, err := types.NewURLsMap(opts.restorationConfig.InitialCluster)
+			if err != nil {
+				logger.Fatalf("failed creating url map for restore cluster: %v", err)
+			}
+
+			peerUrls, err := types.NewURLs(opts.restorationConfig.InitialAdvertisePeerURLs)
+			if err != nil {
+				logger.Fatalf("failed parsing peers urls for restore cluster: %v", err)
+			}
+
+			store, err := snapstore.GetSnapstore(opts.snapstoreConfig)
+			if err != nil {
+				logger.Fatalf("failed to create restore snapstore from configured storage provider: %v", err)
+			}
+
+			logger.Info("Finding latest set of snapshot to recover from...")
+			baseSnap, deltaSnapList, err := miscellaneous.GetLatestFullSnapshotAndDeltaSnapList(store)
+			if err != nil {
+				logger.Fatalf("failed to get latest snapshot: %v", err)
+			}
+			if baseSnap == nil {
+				logger.Infof("No snapshot found. Will do nothing.")
+				return
+			}
+
+			cp := compactor.NewCompactor(store, logrus.NewEntry(logger))
+
+			options := &brtypes.RestoreOptions{
+				Config:        opts.restorationConfig,
+				BaseSnapshot:  baseSnap,
+				DeltaSnapList: deltaSnapList,
+				ClusterURLs:   clusterUrlsMap,
+				PeerURLs:      peerUrls,
+			}
+
+			res, err := cp.Compact(options, opts.needDefragmentation)
+			if err != nil {
+				logger.Fatalf("Failed to restore snapshot: %v", err)
+				return
+			}
+			logger.Infof("Compacted snapshot is in: %v", filepath.Join(opts.snapstoreConfig.Container, res.Snapshot.SnapDir, res.Snapshot.SnapName))
+		},
+	}
+
+	opts.addFlags(compactCmd.Flags())
+	return compactCmd
+}

cmd/initializer.go

@@ -20,7 +20,7 @@ import (
 
 	"github.com/gardener/etcd-backup-restore/pkg/initializer"
 	"github.com/gardener/etcd-backup-restore/pkg/initializer/validator"
-	"github.com/gardener/etcd-backup-restore/pkg/snapshot/restorer"
+	brtypes "github.com/gardener/etcd-backup-restore/pkg/types"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"go.etcd.io/etcd/pkg/types"
@@ -64,7 +64,7 @@ func NewInitializeCommand(ctx context.Context) *cobra.Command {
 				logger.Fatal("validation-mode can only be one of these values [full/sanity]")
 			}
 
-			restoreOptions := &restorer.RestoreOptions{
+			restoreOptions := &brtypes.RestoreOptions{
 				Config:      opts.restorerOptions.restorationConfig,
 				ClusterURLs: clusterUrlsMap,
 				PeerURLs:    peerUrls,

cmd/options.go

@@ -26,14 +26,18 @@ import (
 	"github.com/gardener/etcd-backup-restore/pkg/snapstore"
 
 	"github.com/gardener/etcd-backup-restore/pkg/initializer/validator"
-	"github.com/gardener/etcd-backup-restore/pkg/snapshot/restorer"
 
 	"github.com/gardener/etcd-backup-restore/pkg/server"
+	brtypes "github.com/gardener/etcd-backup-restore/pkg/types"
 	"github.com/ghodss/yaml"
 	"github.com/sirupsen/logrus"
 	flag "github.com/spf13/pflag"
 )
 
+const (
+	defaultCompactContainer = "compctedsnap.bkp"
+)
+
 type serverOptions struct {
 	ConfigFile string
 	Version    bool
@@ -127,15 +131,55 @@ func (c *initializerOptions) complete() {
 	c.restorerOptions.complete()
 }
 
+type compactOptions struct {
+	restorationConfig   *brtypes.RestorationConfig
+	snapstoreConfig     *snapstore.Config
+	needDefragmentation bool
+}
+
+// newCompactOptions returns the validation config.
+func newCompactOptions() *compactOptions {
+	return &compactOptions{
+		restorationConfig:   brtypes.NewRestorationConfig(),
+		snapstoreConfig:     snapstore.NewSnapstoreConfig(),
+		needDefragmentation: true,
+	}
+}
+
+// AddFlags adds the flags to flagset.
+func (c *compactOptions) addFlags(fs *flag.FlagSet) {
+	c.restorationConfig.AddFlags(fs)
+	c.snapstoreConfig.AddFlags(fs)
+	fs.BoolVar(&c.needDefragmentation, "defragment", c.needDefragmentation, "defragment after compaction")
+}
+
+// Validate validates the config.
+func (c *compactOptions) validate() error {
+	if err := c.snapstoreConfig.Validate(); err != nil {
+		return err
+	}
+
+	if err := c.snapstoreConfig.Validate(); err != nil {
+		return err
+	}
+
+	return c.restorationConfig.Validate()
+}
+
+// complete completes the config.
+func (c *compactOptions) complete() {
+	c.snapstoreConfig.Complete()
+}
+
 type restorerOptions struct {
-	restorationConfig *restorer.RestorationConfig
+	restorationConfig *brtypes.RestorationConfig
 	snapstoreConfig   *snapstore.Config
 }
 
 // newRestorerOptions returns the validation config.
 func newRestorerOptions() *restorerOptions {
 	return &restorerOptions{
-		restorationConfig: restorer.NewRestorationConfig(),
+		restorationConfig: brtypes.NewRestorationConfig(),
 		snapstoreConfig:   snapstore.NewSnapstoreConfig(),
 	}
 }

cmd/restore.go

@@ -21,6 +21,7 @@ import (
 	"github.com/gardener/etcd-backup-restore/pkg/miscellaneous"
 	"github.com/gardener/etcd-backup-restore/pkg/snapshot/restorer"
 	"github.com/gardener/etcd-backup-restore/pkg/snapstore"
+	brtypes "github.com/gardener/etcd-backup-restore/pkg/types"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"go.etcd.io/etcd/pkg/types"
@@ -74,9 +75,9 @@ func NewRestoreCommand(ctx context.Context) *cobra.Command {
 
 			rs := restorer.NewRestorer(store, logrus.NewEntry(logger))
 
-			options := &restorer.RestoreOptions{
+			options := &brtypes.RestoreOptions{
 				Config:        opts.restorationConfig,
-				BaseSnapshot:  *baseSnap,
+				BaseSnapshot:  baseSnap,
 				DeltaSnapList: deltaSnapList,
 				ClusterURLs:   clusterUrlsMap,
 				PeerURLs:      peerUrls,

cmd/root.go

@@ -38,6 +38,7 @@ from previously taken snapshot.`,
 	RootCmd.Flags().BoolVarP(&version, "version", "v", false, "print version info")
 	RootCmd.AddCommand(NewSnapshotCommand(ctx),
 		NewRestoreCommand(ctx),
+		NewCompactCommand(ctx),
 		NewInitializeCommand(ctx),
 		NewServerCommand(ctx))
 	return RootCmd