Added verify command (#119)

cmd/knoxite/verify.go

@@ -0,0 +1,181 @@
+/*
+ * knoxite
+ *     Copyright (c) 2020, Fabian Siegel <fabians1999@gmail.com>
+ *
+ *   For license see LICENSE
+ */
+
+package main
+
+import (
+	"fmt"
+
+	"github.com/knoxite/knoxite"
+	"github.com/muesli/goprogressbar"
+	"github.com/spf13/cobra"
+	"github.com/spf13/pflag"
+)
+
+type VerifyOptions struct {
+	Percentage int
+}
+
+var (
+	verifyOpts = VerifyOptions{}
+
+	verifyCmd = &cobra.Command{
+		Use:   "verify [<volume> [<snapshot>]]",
+		Short: "verify a repo, volume or snapshot",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			if len(args) == 0 {
+				return executeVerifyRepo(verifyOpts)
+			} else if len(args) == 1 {
+				return executeVerifyVolume(args[0], verifyOpts)
+			} else if len(args) == 2 {
+				return executeVerifySnapshot(args[0], args[1], verifyOpts)
+			}
+			return nil
+		},
+	}
+)
+
+func initVerifyFlags(f func() *pflag.FlagSet) {
+	f().IntVar(&verifyOpts.Percentage, "percentage", 70, "How many archives to be checked between 0 and 100")
+}
+
+func init() {
+	initVerifyFlags(verifyCmd.Flags)
+	RootCmd.AddCommand(verifyCmd)
+}
+
+func executeVerifyRepo(opts VerifyOptions) error {
+	errors := make([]error, 0)
+	repository, err := openRepository(globalOpts.Repo, globalOpts.Password)
+	if err == nil {
+		progress, err := knoxite.VerifyRepo(repository, opts.Percentage)
+		if err != nil {
+			errors = append(errors, err)
+			return err
+		}
+
+		pb := &goprogressbar.ProgressBar{Total: 1000, Width: 40}
+		lastPath := ""
+
+		for p := range progress {
+			if p.Error != nil {
+				fmt.Println()
+				errors = append(errors, p.Error)
+			}
+
+			pb.Total = int64(p.CurrentItemStats.Size)
+			pb.Current = int64(p.CurrentItemStats.Transferred)
+			pb.PrependText = fmt.Sprintf("%s / %s",
+				knoxite.SizeToString(uint64(pb.Current)),
+				knoxite.SizeToString(uint64(pb.Total)))
+
+			if p.Path != lastPath {
+				// We have just started restoring a new item
+				if len(lastPath) > 0 {
+					fmt.Println()
+				}
+				lastPath = p.Path
+				pb.Text = p.Path
+			}
+
+			pb.LazyPrint()
+
+		}
+		fmt.Println()
+		fmt.Printf("Verify done: %d errors\n", len(errors))
+		return nil
+	}
+	return err
+}
+
+func executeVerifyVolume(volumeId string, opts VerifyOptions) error {
+	errors := make([]error, 0)
+	repository, err := openRepository(globalOpts.Repo, globalOpts.Password)
+	if err == nil {
+		progress, err := knoxite.VerifyVolume(repository, volumeId, opts.Percentage)
+		if err != nil {
+			errors = append(errors, err)
+			return err
+		}
+
+		pb := &goprogressbar.ProgressBar{Total: 1000, Width: 40}
+		lastPath := ""
+
+		for p := range progress {
+			if p.Error != nil {
+				fmt.Println()
+				errors = append(errors, p.Error)
+			}
+
+			pb.Total = int64(p.CurrentItemStats.Size)
+			pb.Current = int64(p.CurrentItemStats.Transferred)
+			pb.PrependText = fmt.Sprintf("%s / %s",
+				knoxite.SizeToString(uint64(pb.Current)),
+				knoxite.SizeToString(uint64(pb.Total)))
+
+			if p.Path != lastPath {
+				// We have just started restoring a new item
+				if len(lastPath) > 0 {
+					fmt.Println()
+				}
+				lastPath = p.Path
+				pb.Text = p.Path
+			}
+
+			pb.LazyPrint()
+
+		}
+		fmt.Println()
+		fmt.Printf("Verify done: %d errors\n", len(errors))
+		return nil
+	}
+	return err
+}
+
+func executeVerifySnapshot(volumeId string, snapshotId string, opts VerifyOptions) error {
+	errors := make([]error, 0)
+	repository, err := openRepository(globalOpts.Repo, globalOpts.Password)
+	if err == nil {
+		progress, err := knoxite.VerifySnapshot(repository, snapshotId, opts.Percentage)
+		if err != nil {
+			errors = append(errors, err)
+			return err
+		}
+
+		pb := &goprogressbar.ProgressBar{Total: 1000, Width: 40}
+		lastPath := ""
+
+		for p := range progress {
+			if p.Error != nil {
+				fmt.Println()
+				errors = append(errors, p.Error)
+			}
+
+			pb.Total = int64(p.CurrentItemStats.Size)
+			pb.Current = int64(p.CurrentItemStats.Transferred)
+			pb.PrependText = fmt.Sprintf("%s / %s",
+				knoxite.SizeToString(uint64(pb.Current)),
+				knoxite.SizeToString(uint64(pb.Total)))
+
+			if p.Path != lastPath {
+				// We have just started restoring a new item
+				if len(lastPath) > 0 {
+					fmt.Println()
+				}
+				lastPath = p.Path
+				pb.Text = p.Path
+			}
+
+			pb.LazyPrint()
+
+		}
+		fmt.Println()
+		fmt.Printf("Verify done: %d errors\n", len(errors))
+		return nil
+	}
+	return err
+}

verify.go

@@ -0,0 +1,207 @@
+/*
+ * knoxite
+ *     Copyright (c) 2020, Fabian Siegel <fabians1999@gmail.com>
+ *
+ *   For license see LICENSE
+ */
+
+package knoxite
+
+import (
+	"math"
+	"math/rand"
+)
+
+func VerifyRepo(repository Repository, percentage int) (prog chan Progress, err error) {
+	prog = make(chan Progress)
+
+	go func() {
+		archiveToSnapshot := make(map[string]*Snapshot)
+
+		for _, volume := range repository.Volumes {
+			for _, snapshotHash := range volume.Snapshots {
+				_, snapshot, err := repository.FindSnapshot(snapshotHash)
+				if err != nil {
+					prog <- newProgressError(err)
+				}
+
+				for archiveHash := range snapshot.Archives {
+					archiveToSnapshot[archiveHash] = snapshot
+				}
+			}
+		}
+
+		archives := make([]string, 0)
+		for archiveHash := range archiveToSnapshot {
+			archives = append(archives, archiveHash)
+		}
+
+		// get all keys of the snapshot Archives
+		if percentage > 100 {
+			percentage = 100
+		} else if percentage < 0 {
+			percentage = 0
+		}
+
+		// select len(keys)*percentage unique keys to verify
+		nrOfSelectedArchives := int(math.Ceil(float64(len(archives)*percentage) / 100.0))
+
+		// we use a map[string]bool as a Set implementation
+		selectedArchives := make(map[string]bool)
+		for len(selectedArchives) < nrOfSelectedArchives {
+			idx := rand.Intn(len(archives))
+			selectedArchives[archives[idx]] = true
+		}
+
+		for archiveKey := range selectedArchives {
+			snapshot := archiveToSnapshot[archiveKey]
+			p := newProgress(snapshot.Archives[archiveKey])
+			prog <- p
+
+			err := VerifyArchive(repository, *snapshot.Archives[archiveKey])
+			if err != nil {
+				prog <- newProgressError(err)
+			}
+
+			p.CurrentItemStats.Transferred += uint64((*snapshot.Archives[archiveKey]).Size)
+			prog <- p
+
+		}
+		close(prog)
+	}()
+
+	return prog, nil
+}
+
+func VerifyVolume(repository Repository, volumeId string, percentage int) (prog chan Progress, err error) {
+	prog = make(chan Progress)
+
+	go func() {
+		volume, ferr := repository.FindVolume(volumeId)
+		if ferr != nil {
+			prog <- newProgressError(ferr)
+		}
+
+		archiveToSnapshot := make(map[string]*Snapshot)
+
+		for _, snapshotHash := range volume.Snapshots {
+			_, snapshot, err := repository.FindSnapshot(snapshotHash)
+			if err != nil {
+				prog <- newProgressError(err)
+			}
+
+			for archiveHash := range snapshot.Archives {
+				archiveToSnapshot[archiveHash] = snapshot
+			}
+		}
+
+		archives := make([]string, 0)
+		for archiveHash := range archiveToSnapshot {
+			archives = append(archives, archiveHash)
+		}
+
+		// get all keys of the snapshot Archives
+		if percentage > 100 {
+			percentage = 100
+		} else if percentage < 0 {
+			percentage = 0
+		}
+
+		// select len(keys)*percentage unique keys to verify
+		nrOfSelectedArchives := int(math.Ceil(float64(len(archives)*percentage) / 100.0))
+
+		// we use a map[string]bool as a Set implementation
+		selectedArchives := make(map[string]bool)
+		for len(selectedArchives) < nrOfSelectedArchives {
+			idx := rand.Intn(len(archives))
+			selectedArchives[archives[idx]] = true
+		}
+
+		for archiveKey := range selectedArchives {
+			snapshot := archiveToSnapshot[archiveKey]
+			p := newProgress(snapshot.Archives[archiveKey])
+			prog <- p
+
+			err := VerifyArchive(repository, *snapshot.Archives[archiveKey])
+			if err != nil {
+				prog <- newProgressError(err)
+			}
+
+			p.CurrentItemStats.Transferred += uint64((*snapshot.Archives[archiveKey]).Size)
+			prog <- p
+		}
+		close(prog)
+	}()
+
+	return prog, nil
+}
+
+func VerifySnapshot(repository Repository, snapshotId string, percentage int) (prog chan Progress, err error) {
+	prog = make(chan Progress)
+
+	go func() {
+		_, snapshot, ferr := repository.FindSnapshot(snapshotId)
+		if ferr != nil {
+			prog <- newProgressError(ferr)
+		}
+
+		// get all keys of the snapshot Archives
+		archives := make([]string, 0, len(snapshot.Archives))
+		for key := range snapshot.Archives {
+			archives = append(archives, key)
+		}
+
+		if percentage > 100 {
+			percentage = 100
+		} else if percentage < 0 {
+			percentage = 0
+		}
+
+		// select len(keys)*percentage unique keys to verify
+		nrOfSelectedArchives := int(math.Ceil(float64(len(archives)*percentage) / 100.0))
+
+		// we use a map[string]bool as a Set implementation
+		selectedArchives := make(map[string]bool)
+		for len(selectedArchives) < nrOfSelectedArchives {
+			idx := rand.Intn(len(archives))
+			selectedArchives[archives[idx]] = true
+		}
+
+		for archiveKey := range selectedArchives {
+			p := newProgress(snapshot.Archives[archiveKey])
+			prog <- p
+
+			err := VerifyArchive(repository, *snapshot.Archives[archiveKey])
+			if err != nil {
+				prog <- newProgressError(err)
+			}
+
+			p.CurrentItemStats.Transferred += uint64((*snapshot.Archives[archiveKey]).Size)
+			prog <- p
+
+		}
+		close(prog)
+	}()
+
+	return prog, nil
+}
+
+func VerifyArchive(repository Repository, arc Archive) error {
+	if arc.Type == File {
+		parts := uint(len(arc.Chunks))
+		for i := uint(0); i < parts; i++ {
+			idx, erri := arc.IndexOfChunk(i)
+			if erri != nil {
+				return erri
+			}
+
+			chunk := arc.Chunks[idx]
+			_, errc := loadChunk(repository, arc, chunk)
+			if errc != nil {
+				return errc
+			}
+		}
+		return nil
+	}
+	return nil
+}