diff --git a/lib/archive.go b/archive.go similarity index 97% rename from lib/archive.go rename to archive.go index f23759e9..8e5ca6d0 100644 --- a/lib/archive.go +++ b/archive.go @@ -1,6 +1,6 @@ /* * knoxite - * Copyright (c) 2016-2018, Christian Muehlhaeuser + * Copyright (c) 2016-2020, Christian Muehlhaeuser * * For license see LICENSE */ diff --git a/lib/backend.go b/backend.go similarity index 100% rename from lib/backend.go rename to backend.go diff --git a/lib/backend_test.go b/backend_test.go similarity index 100% rename from lib/backend_test.go rename to backend_test.go diff --git a/lib/backendmanager.go b/backendmanager.go similarity index 100% rename from lib/backendmanager.go rename to backendmanager.go diff --git a/lib/chunk.go b/chunk.go similarity index 100% rename from lib/chunk.go rename to chunk.go diff --git a/lib/chunkindex.go b/chunkindex.go similarity index 100% rename from lib/chunkindex.go rename to chunkindex.go diff --git a/lib/chunkindex_test.go b/chunkindex_test.go similarity index 100% rename from lib/chunkindex_test.go rename to chunkindex_test.go diff --git a/cat.go b/cmd/knoxite/cat.go similarity index 91% rename from cat.go rename to cmd/knoxite/cat.go index 9259df5b..a50c568d 100644 --- a/cat.go +++ b/cmd/knoxite/cat.go @@ -1,6 +1,6 @@ /* * knoxite - * Copyright (c) 2016-2018, Christian Muehlhaeuser + * Copyright (c) 2016-2020, Christian Muehlhaeuser * * For license see LICENSE */ @@ -11,7 +11,7 @@ import ( "fmt" "os" - "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" "github.com/spf13/cobra" ) diff --git a/clone.go b/cmd/knoxite/clone.go similarity index 93% rename from clone.go rename to cmd/knoxite/clone.go index 211dada6..196ce64c 100644 --- a/clone.go +++ b/cmd/knoxite/clone.go @@ -1,6 +1,6 @@ /* * knoxite - * Copyright (c) 2016-2017, Christian Muehlhaeuser + * Copyright (c) 2016-2020, Christian Muehlhaeuser * * For license see LICENSE */ @@ -11,10 +11,10 @@ import ( "fmt" "path/filepath" - "github.com/klauspost/shutdown2" + shutdown "github.com/klauspost/shutdown2" "github.com/spf13/cobra" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) var ( diff --git a/ls.go b/cmd/knoxite/ls.go similarity index 93% rename from ls.go rename to cmd/knoxite/ls.go index 030190d1..341f3f09 100644 --- a/ls.go +++ b/cmd/knoxite/ls.go @@ -1,6 +1,6 @@ /* * knoxite - * Copyright (c) 2016-2018, Christian Muehlhaeuser + * Copyright (c) 2016-2020, Christian Muehlhaeuser * * For license see LICENSE */ @@ -16,7 +16,7 @@ import ( "github.com/muesli/gotable" "github.com/spf13/cobra" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) const timeFormat = "2006-01-02 15:04:05" diff --git a/main.go b/cmd/knoxite/main.go similarity index 100% rename from main.go rename to cmd/knoxite/main.go diff --git a/mount.go b/cmd/knoxite/mount.go similarity index 98% rename from mount.go rename to cmd/knoxite/mount.go index 2e6097c9..bfa80780 100644 --- a/mount.go +++ b/cmd/knoxite/mount.go @@ -3,7 +3,7 @@ /* * knoxite - * Copyright (c) 2016-2017, Christian Muehlhaeuser + * Copyright (c) 2016-2020, Christian Muehlhaeuser * * For license see LICENSE */ @@ -23,7 +23,7 @@ import ( "github.com/spf13/cobra" "golang.org/x/net/context" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) var ( diff --git a/cmd/knoxite/repository.go b/cmd/knoxite/repository.go new file mode 100644 index 00000000..7e7090e6 --- /dev/null +++ b/cmd/knoxite/repository.go @@ -0,0 +1,267 @@ +/* + * knoxite + * Copyright (c) 2016-2020, Christian Muehlhaeuser + * + * For license see LICENSE + */ + +package main + +import ( + "encoding/json" + "errors" + "fmt" + "io" + "os" + "strings" + "syscall" + + shutdown "github.com/klauspost/shutdown2" + "github.com/muesli/crunchy" + "github.com/muesli/gotable" + "github.com/spf13/cobra" + "golang.org/x/crypto/ssh/terminal" + + "github.com/knoxite/knoxite" +) + +// Error declarations +var ( + ErrPasswordMismatch = errors.New("Passwords did not match") + + repoCmd = &cobra.Command{ + Use: "repo", + Short: "manage repository", + Long: `The repo command manages repositories`, + RunE: nil, + } + repoInitCmd = &cobra.Command{ + Use: "init", + Short: "initialize a new repository", + Long: `The init command initializes a new repository`, + RunE: func(cmd *cobra.Command, args []string) error { + return executeRepoInit() + }, + } + repoCatCmd = &cobra.Command{ + Use: "cat", + Short: "display repository information as JSON", + Long: `The cat command displays the internal repository information as JSON`, + RunE: func(cmd *cobra.Command, args []string) error { + return executeRepoCat() + }, + } + repoInfoCmd = &cobra.Command{ + Use: "info", + Short: "display repository information", + Long: `The info command displays the repository status & information`, + RunE: func(cmd *cobra.Command, args []string) error { + return executeRepoInfo() + }, + } + repoAddCmd = &cobra.Command{ + Use: "add ", + Short: "add another storage backend to a repository", + Long: `The add command adds another storage backend to a repository`, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("add needs a URL to be added") + } + return executeRepoAdd(args[0]) + }, + } + repoPackCmd = &cobra.Command{ + Use: "pack", + Short: "pack repository and release redundant data", + Long: `The pack command deletes all unused data chunks from storage`, + RunE: func(cmd *cobra.Command, args []string) error { + return executeRepoPack() + }, + } +) + +func init() { + repoCmd.AddCommand(repoInitCmd) + repoCmd.AddCommand(repoCatCmd) + repoCmd.AddCommand(repoInfoCmd) + repoCmd.AddCommand(repoAddCmd) + repoCmd.AddCommand(repoPackCmd) + RootCmd.AddCommand(repoCmd) +} + +func executeRepoInit() error { + // acquire a shutdown lock. we don't want these next calls to be interrupted + lock := shutdown.Lock() + if lock == nil { + return nil + } + defer lock() + + r, err := newRepository(globalOpts.Repo, globalOpts.Password) + if err != nil { + return fmt.Errorf("Creating repository at %s failed: %v", globalOpts.Repo, err) + } + + fmt.Printf("Created new repository at %s\n", (*r.BackendManager().Backends[0]).Location()) + return nil +} + +func executeRepoAdd(url string) error { + // acquire a shutdown lock. we don't want these next calls to be interrupted + lock := shutdown.Lock() + if lock == nil { + return nil + } + defer lock() + + r, err := openRepository(globalOpts.Repo, globalOpts.Password) + if err != nil { + return err + } + + backend, err := knoxite.BackendFromURL(url) + if err != nil { + return err + } + r.BackendManager().AddBackend(&backend) + + err = r.Save() + if err != nil { + return err + } + fmt.Printf("Added %s to repository\n", backend.Location()) + return nil +} + +func executeRepoCat() error { + r, err := openRepository(globalOpts.Repo, globalOpts.Password) + if err != nil { + return err + } + + json, err := json.MarshalIndent(r, "", " ") + if err != nil { + return err + } + fmt.Printf("%s\n", json) + return nil +} + +func executeRepoPack() error { + r, err := openRepository(globalOpts.Repo, globalOpts.Password) + if err != nil { + return err + } + index, err := knoxite.OpenChunkIndex(&r) + if err != nil { + return err + } + + freedSize, err := index.Pack(&r) + if err != nil { + return err + } + + err = index.Save(&r) + if err != nil { + return err + } + + fmt.Printf("Freed storage space: %s\n", knoxite.SizeToString(freedSize)) + return nil +} + +func executeRepoInfo() error { + r, err := openRepository(globalOpts.Repo, globalOpts.Password) + if err != nil { + return err + } + + tab := gotable.NewTable([]string{"Storage URL", "Available Space"}, + []int64{-48, 15}, + "No backends found.") + + for _, be := range r.BackendManager().Backends { + space, _ := (*be).AvailableSpace() + tab.AppendRow([]interface{}{ + (*be).Location(), + knoxite.SizeToString(space)}) + } + + _ = tab.Print() + return nil +} + +func openRepository(path, password string) (knoxite.Repository, error) { + if password == "" { + var err error + password, err = readPassword("Enter password:") + if err != nil { + return knoxite.Repository{}, err + } + } + + return knoxite.OpenRepository(path, password) +} + +func newRepository(path, password string) (knoxite.Repository, error) { + if password == "" { + var err error + password, err = readPasswordTwice("Enter a password to encrypt this repository with:", "Confirm password:") + if err != nil { + return knoxite.Repository{}, err + } + } + + return knoxite.NewRepository(path, password) +} + +func readPassword(prompt string) (string, error) { + var tty io.WriteCloser + tty, err := os.OpenFile("/dev/tty", os.O_WRONLY, 0) + if err != nil { + tty = os.Stdout + } else { + defer tty.Close() + } + + fmt.Fprint(tty, prompt+" ") + buf, err := terminal.ReadPassword(int(syscall.Stdin)) + fmt.Fprintln(tty) + + return string(buf), err +} + +func readPasswordTwice(prompt, promptConfirm string) (string, error) { + pw, err := readPassword(prompt) + if err != nil { + return pw, err + } + + crunchErr := crunchy.NewValidator().Check(pw) + if crunchErr != nil { + fmt.Printf("Password is considered unsafe: %v\n", crunchErr) + fmt.Printf("Are you sure you want to use this password (y/N)?: ") + var buf string + _, err = fmt.Scan(&buf) + if err != nil { + return pw, err + } + + buf = strings.TrimSpace(buf) + buf = strings.ToLower(buf) + if buf != "y" { + return pw, crunchErr + } + } + + pwconfirm, err := readPassword(promptConfirm) + if err != nil { + return pw, err + } + if pw != pwconfirm { + return pw, ErrPasswordMismatch + } + + return pw, nil +} diff --git a/restore.go b/cmd/knoxite/restore.go similarity index 95% rename from restore.go rename to cmd/knoxite/restore.go index fcf37c51..f4554617 100644 --- a/restore.go +++ b/cmd/knoxite/restore.go @@ -1,6 +1,6 @@ /* * knoxite - * Copyright (c) 2016-2017, Christian Muehlhaeuser + * Copyright (c) 2016-2020, Christian Muehlhaeuser * * For license see LICENSE */ @@ -15,7 +15,7 @@ import ( "github.com/spf13/cobra" "github.com/spf13/pflag" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) // Error declarations diff --git a/cmd/knoxite/snapshot.go b/cmd/knoxite/snapshot.go new file mode 100644 index 00000000..70f60ca0 --- /dev/null +++ b/cmd/knoxite/snapshot.go @@ -0,0 +1,126 @@ +/* + * knoxite + * Copyright (c) 2016-2020, Christian Muehlhaeuser + * + * For license see LICENSE + */ + +package main + +import ( + "fmt" + + "github.com/muesli/gotable" + "github.com/spf13/cobra" + + "github.com/knoxite/knoxite" +) + +var ( + snapshotCmd = &cobra.Command{ + Use: "snapshot", + Short: "manage snapshots", + Long: `The snapshot command manages snapshots`, + RunE: nil, + } + snapshotListCmd = &cobra.Command{ + Use: "list ", + Short: "list all snapshots inside a volume", + Long: `The list command lists all snapshots stored in a volume`, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("list needs a volume ID to work on") + } + return executeSnapshotList(args[0]) + }, + } + snapshotRemoveCmd = &cobra.Command{ + Use: "remove ", + Short: "remove a snapshot", + Long: `The remove command deletes a snapshot from a volume`, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("remove needs a snapshot ID to work on") + } + return executeSnapshotRemove(args[0]) + }, + } +) + +func init() { + snapshotCmd.AddCommand(snapshotListCmd) + snapshotCmd.AddCommand(snapshotRemoveCmd) + RootCmd.AddCommand(snapshotCmd) +} + +func executeSnapshotRemove(snapshotID string) error { + repository, err := openRepository(globalOpts.Repo, globalOpts.Password) + if err != nil { + return err + } + chunkIndex, err := knoxite.OpenChunkIndex(&repository) + if err != nil { + return err + } + + volume, snapshot, err := repository.FindSnapshot(snapshotID) + if err != nil { + return err + } + + err = volume.RemoveSnapshot(snapshot.ID) + if err != nil { + return err + } + + chunkIndex.RemoveSnapshot(snapshot.ID) + err = chunkIndex.Save(&repository) + if err != nil { + return err + } + + err = repository.Save() + if err != nil { + return err + } + + fmt.Printf("Snapshot %s removed: %s\n", snapshot.ID, snapshot.Stats.String()) + fmt.Println("Do not forget to run 'repo pack' to delete un-referenced chunks and free up storage space!") + return nil +} + +func executeSnapshotList(volID string) error { + repository, err := openRepository(globalOpts.Repo, globalOpts.Password) + if err != nil { + return err + } + + volume, err := repository.FindVolume(volID) + if err != nil { + return err + } + + tab := gotable.NewTable([]string{"ID", "Date", "Original Size", "Storage Size", "Description"}, + []int64{-8, -19, 13, 12, -48}, "No snapshots found. This volume is empty.") + totalSize := uint64(0) + totalStorageSize := uint64(0) + + for _, snapshotID := range volume.Snapshots { + snapshot, err := volume.LoadSnapshot(snapshotID, &repository) + if err != nil { + return err + } + tab.AppendRow([]interface{}{ + snapshot.ID, + snapshot.Date.Format(timeFormat), + knoxite.SizeToString(snapshot.Stats.Size), + knoxite.SizeToString(snapshot.Stats.StorageSize), + snapshot.Description}) + totalSize += snapshot.Stats.Size + totalStorageSize += snapshot.Stats.StorageSize + } + + tab.SetSummary([]interface{}{"", "", knoxite.SizeToString(totalSize), knoxite.SizeToString(totalStorageSize), ""}) + _ = tab.Print() + return nil +} diff --git a/store.go b/cmd/knoxite/store.go similarity index 98% rename from store.go rename to cmd/knoxite/store.go index ed4467b1..60c3c588 100644 --- a/store.go +++ b/cmd/knoxite/store.go @@ -1,6 +1,6 @@ /* * knoxite - * Copyright (c) 2016-2018, Christian Muehlhaeuser + * Copyright (c) 2016-2020, Christian Muehlhaeuser * * For license see LICENSE */ @@ -16,12 +16,12 @@ import ( "time" humanize "github.com/dustin/go-humanize" - "github.com/klauspost/shutdown2" + shutdown "github.com/klauspost/shutdown2" "github.com/muesli/goprogressbar" "github.com/spf13/cobra" "github.com/spf13/pflag" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) // Error declarations diff --git a/cmd/knoxite/volume.go b/cmd/knoxite/volume.go new file mode 100644 index 00000000..9d25de74 --- /dev/null +++ b/cmd/knoxite/volume.go @@ -0,0 +1,105 @@ +/* + * knoxite + * Copyright (c) 2016-2020, Christian Muehlhaeuser + * + * For license see LICENSE + */ + +package main + +import ( + "fmt" + + shutdown "github.com/klauspost/shutdown2" + "github.com/muesli/gotable" + "github.com/spf13/cobra" + + "github.com/knoxite/knoxite" +) + +// VolumeInitOptions holds all the options that can be set for the 'volume init' command +type VolumeInitOptions struct { + Description string +} + +var ( + volumeInitOpts = VolumeInitOptions{} + + volumeCmd = &cobra.Command{ + Use: "volume", + Short: "manage volumes", + Long: `The volume command manages volumes`, + RunE: nil, + } + volumeInitCmd = &cobra.Command{ + Use: "init ", + Short: "initialize a new volume", + Long: `The init command initializes a new volume`, + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("init needs a name for the new volume") + } + return executeVolumeInit(args[0], volumeInitOpts.Description) + }, + } + volumeListCmd = &cobra.Command{ + Use: "list", + Short: "list all volumes inside a repository", + Long: `The list command lists all volumes stored in a repository`, + RunE: func(cmd *cobra.Command, args []string) error { + return executeVolumeList() + }, + } +) + +func init() { + volumeInitCmd.Flags().StringVarP(&volumeInitOpts.Description, "desc", "d", "", "a description or comment for this volume") + + volumeCmd.AddCommand(volumeInitCmd) + volumeCmd.AddCommand(volumeListCmd) + RootCmd.AddCommand(volumeCmd) +} + +func executeVolumeInit(name, description string) error { + // acquire a shutdown lock. we don't want these next calls to be interrupted + lock := shutdown.Lock() + if lock == nil { + return nil + } + defer lock() + + repository, err := openRepository(globalOpts.Repo, globalOpts.Password) + if err == nil { + vol, verr := knoxite.NewVolume(name, description) + if verr == nil { + verr = repository.AddVolume(vol) + if verr != nil { + return fmt.Errorf("Creating volume %s failed: %v", name, verr) + } + + annotation := "Name: " + vol.Name + if len(vol.Description) > 0 { + annotation += ", Description: " + vol.Description + } + fmt.Printf("Volume %s (%s) created\n", vol.ID, annotation) + return repository.Save() + } + } + return err +} + +func executeVolumeList() error { + repository, err := openRepository(globalOpts.Repo, globalOpts.Password) + if err != nil { + return err + } + + tab := gotable.NewTable([]string{"ID", "Name", "Description"}, + []int64{-8, -32, -48}, "No volumes found. This repository is empty.") + for _, volume := range repository.Volumes { + tab.AppendRow([]interface{}{volume.ID, volume.Name, volume.Description}) + } + + _ = tab.Print() + return nil +} diff --git a/lib/compression.go b/compression.go similarity index 100% rename from lib/compression.go rename to compression.go diff --git a/lib/decode.go b/decode.go similarity index 100% rename from lib/decode.go rename to decode.go diff --git a/lib/encryption.go b/encryption.go similarity index 100% rename from lib/encryption.go rename to encryption.go diff --git a/lib/encryption_test.go b/encryption_test.go similarity index 100% rename from lib/encryption_test.go rename to encryption_test.go diff --git a/lib/hash.go b/hash.go similarity index 100% rename from lib/hash.go rename to hash.go diff --git a/lib/repository.go b/lib/repository.go deleted file mode 100644 index 102d67d0..00000000 --- a/lib/repository.go +++ /dev/null @@ -1,188 +0,0 @@ -/* - * knoxite - * Copyright (c) 2016-2018, Christian Muehlhaeuser - * - * For license see LICENSE - */ - -package knoxite - -import ( - "errors" -) - -// A Repository is a collection of backup snapshots -// MUST BE encrypted -type Repository struct { - Version uint `json:"version"` - Volumes []*Volume `json:"volumes"` - Paths []string `json:"storage"` - // Owner string `json:"owner"` - - backend BackendManager - password string -} - -// Const declarations -const ( - RepositoryVersion = 3 -) - -// Error declarations -var ( - ErrRepositoryIncompatible = errors.New("The repository is not compatible with this version of Knoxite") - ErrOpenRepositoryFailed = errors.New("Wrong password or corrupted repository") - ErrVolumeNotFound = errors.New("Volume not found") - ErrSnapshotNotFound = errors.New("Snapshot not found") -) - -// NewRepository returns a new repository -func NewRepository(path, password string) (Repository, error) { - repository := Repository{ - Version: RepositoryVersion, - password: password, - } - - backend, err := BackendFromURL(path) - if err != nil { - return repository, err - } - repository.backend.AddBackend(&backend) - - err = repository.init() - return repository, err -} - -// OpenRepository opens an existing repository -func OpenRepository(path, password string) (Repository, error) { - repository := Repository{ - password: password, - } - - backend, err := BackendFromURL(path) - if err != nil { - return repository, err - } - b, err := backend.LoadRepository() - if err != nil { - return repository, err - } - - pipe, err := NewDecodingPipeline(CompressionNone, EncryptionAES, password) - if err != nil { - return repository, err - } - err = pipe.Decode(b, &repository) - if err != nil { - return repository, ErrOpenRepositoryFailed - } - if repository.Version < RepositoryVersion { - return repository, ErrRepositoryIncompatible - } - - for _, url := range repository.Paths { - backend, berr := BackendFromURL(url) - if berr != nil { - return repository, berr - } - repository.backend.AddBackend(&backend) - } - - return repository, err -} - -// AddVolume adds a volume to a repository -func (r *Repository) AddVolume(volume *Volume) error { - r.Volumes = append(r.Volumes, volume) - return nil -} - -// FindVolume finds a volume within a repository -func (r *Repository) FindVolume(id string) (*Volume, error) { - if id == "latest" && len(r.Volumes) > 0 { - return r.Volumes[len(r.Volumes)-1], nil - } - - for _, volume := range r.Volumes { - if volume.ID == id { - return volume, nil - } - } - - return &Volume{}, ErrVolumeNotFound -} - -// FindSnapshot finds a snapshot within a repository -func (r *Repository) FindSnapshot(id string) (*Volume, *Snapshot, error) { - if id == "latest" { - latestVolume := &Volume{} - latestSnapshot := &Snapshot{} - found := false - for _, volume := range r.Volumes { - for _, snapshotID := range volume.Snapshots { - snapshot, err := volume.LoadSnapshot(snapshotID, r) - if err == nil { - if !found || snapshot.Date.Sub(latestSnapshot.Date) > 0 { - latestSnapshot = snapshot - latestVolume = volume - found = true - } - } - } - } - - if found { - return latestVolume, latestSnapshot, nil - } - } else { - for _, volume := range r.Volumes { - snapshot, err := volume.LoadSnapshot(id, r) - if err == nil { - return volume, snapshot, err - } - } - } - - return &Volume{}, &Snapshot{}, ErrSnapshotNotFound -} - -// IsEmpty returns true if there a no snapshots stored in a repository -func (r *Repository) IsEmpty() bool { - for _, volume := range r.Volumes { - if len(volume.Snapshots) > 0 { - return false - } - } - - return true -} - -// BackendManager returns the repository's BackendManager -func (r *Repository) BackendManager() *BackendManager { - return &r.backend -} - -// Init creates a new repository -func (r *Repository) init() error { - err := r.backend.InitRepository() - if err != nil { - return err - } - - return r.Save() -} - -// Save writes a repository's metadata -func (r *Repository) Save() error { - r.Paths = r.backend.Locations() - - pipe, err := NewEncodingPipeline(CompressionNone, EncryptionAES, r.password) - if err != nil { - return err - } - b, err := pipe.Encode(r) - if err != nil { - return err - } - return r.backend.SaveRepository(b) -} diff --git a/lib/snapshot.go b/lib/snapshot.go deleted file mode 100644 index fb004af6..00000000 --- a/lib/snapshot.go +++ /dev/null @@ -1,227 +0,0 @@ -/* - * knoxite - * Copyright (c) 2016-2018, Christian Muehlhaeuser - * - * For license see LICENSE - */ - -package knoxite - -import ( - "math" - "os" - "path/filepath" - "strings" - "sync" - "time" - - uuid "github.com/nu7hatch/gouuid" -) - -// A Snapshot is a compilation of one or many archives -// MUST BE encrypted -type Snapshot struct { - mut sync.Mutex - - ID string `json:"id"` - Date time.Time `json:"date"` - Description string `json:"description"` - Stats Stats `json:"stats"` - Archives map[string]*Archive `json:"items"` -} - -// NewSnapshot creates a new snapshot -func NewSnapshot(description string) (*Snapshot, error) { - snapshot := Snapshot{ - Date: time.Now(), - Description: description, - Archives: make(map[string]*Archive), - } - - u, err := uuid.NewV4() - if err != nil { - return &snapshot, err - } - snapshot.ID = u.String()[:8] - - return &snapshot, nil -} - -func (snapshot *Snapshot) gatherTargetInformation(cwd string, paths []string, excludes []string, out chan ArchiveResult) { - var wg sync.WaitGroup - for _, path := range paths { - c := findFiles(path, excludes) - - for result := range c { - if result.Error == nil { - rel, err := filepath.Rel(cwd, result.Archive.Path) - if err == nil && !strings.HasPrefix(rel, "../") { - result.Archive.Path = rel - } - if isSpecialPath(result.Archive.Path) { - continue - } - - snapshot.mut.Lock() - snapshot.Stats.Size += result.Archive.Size - switch result.Archive.Type { - case Directory: - snapshot.Stats.Dirs++ - case File: - snapshot.Stats.Files++ - case SymLink: - snapshot.Stats.SymLinks++ - } - snapshot.mut.Unlock() - } - - wg.Add(1) - go func(r ArchiveResult) { - out <- r - wg.Done() - }(result) - } - } - - wg.Wait() - close(out) -} - -// Add adds a path to a Snapshot -func (snapshot *Snapshot) Add(cwd string, paths []string, excludes []string, repository Repository, chunkIndex *ChunkIndex, compress, encrypt uint16, dataParts, parityParts uint) chan Progress { - progress := make(chan Progress) - fwd := make(chan ArchiveResult) - - go snapshot.gatherTargetInformation(cwd, paths, excludes, fwd) - - go func() { - for result := range fwd { - if result.Error != nil { - p := newProgressError(result.Error) - progress <- p - break - } - - archive := result.Archive - rel, err := filepath.Rel(cwd, archive.Path) - if err == nil && !strings.HasPrefix(rel, "../") { - archive.Path = rel - } - if isSpecialPath(archive.Path) { - continue - } - - p := newProgress(archive) - snapshot.mut.Lock() - p.TotalStatistics = snapshot.Stats - snapshot.mut.Unlock() - progress <- p - - if archive.Type == File { - dataParts = uint(math.Max(1, float64(dataParts))) - chunkchan, err := chunkFile(archive.Path, compress, encrypt, repository.password, int(dataParts), int(parityParts)) - if err != nil { - if os.IsNotExist(err) { - // if this file has already been deleted before we could backup it, we can gracefully ignore it and continue - continue - } - p = newProgressError(err) - progress <- p - break - } - archive.Encrypted = encrypt - archive.Compressed = compress - - for cd := range chunkchan { - if cd.Error != nil { - p = newProgressError(err) - progress <- p - close(progress) - return - } - chunk := cd.Chunk - // fmt.Printf("\tSplit %s (#%d, %d bytes), compression: %s, encryption: %s, hash: %s\n", id.Path, cd.Num, cd.Size, CompressionText(cd.Compressed), EncryptionText(cd.Encrypted), cd.Hash) - - // store this chunk - n, err := repository.backend.StoreChunk(chunk) - if err != nil { - p = newProgressError(err) - progress <- p - close(progress) - return - } - - // release the memory, we don't need the data anymore - chunk.Data = &[][]byte{} - - archive.Chunks = append(archive.Chunks, chunk) - archive.StorageSize += n - - p.CurrentItemStats.StorageSize = archive.StorageSize - p.CurrentItemStats.Transferred += uint64(chunk.OriginalSize) - snapshot.Stats.Transferred += uint64(chunk.OriginalSize) - snapshot.Stats.StorageSize += n - - snapshot.mut.Lock() - p.TotalStatistics = snapshot.Stats - snapshot.mut.Unlock() - progress <- p - } - } - - snapshot.AddArchive(archive) - chunkIndex.AddArchive(archive, snapshot.ID) - } - close(progress) - }() - - return progress -} - -// Clone clones a snapshot -func (snapshot *Snapshot) Clone() (*Snapshot, error) { - s, err := NewSnapshot(snapshot.Description) - if err != nil { - return s, err - } - - s.Stats = snapshot.Stats - s.Archives = snapshot.Archives - - return s, nil -} - -// openSnapshot opens an existing snapshot -func openSnapshot(id string, repository *Repository) (*Snapshot, error) { - snapshot := Snapshot{ - Archives: make(map[string]*Archive), - } - b, err := repository.backend.LoadSnapshot(id) - if err != nil { - return &snapshot, err - } - pipe, err := NewDecodingPipeline(CompressionLZMA, EncryptionAES, repository.password) - if err != nil { - return &snapshot, err - } - err = pipe.Decode(b, &snapshot) - return &snapshot, err -} - -// Save writes a snapshot's metadata -func (snapshot *Snapshot) Save(repository *Repository) error { - pipe, err := NewEncodingPipeline(CompressionLZMA, EncryptionAES, repository.password) - if err != nil { - return err - } - b, err := pipe.Encode(snapshot) - if err != nil { - return err - } - return repository.backend.SaveSnapshot(snapshot.ID, b) -} - -// AddArchive adds an archive to a snapshot -func (snapshot *Snapshot) AddArchive(archive *Archive) { - snapshot.Archives[archive.Path] = archive -} diff --git a/lib/volume.go b/lib/volume.go deleted file mode 100644 index f6ba30b3..00000000 --- a/lib/volume.go +++ /dev/null @@ -1,74 +0,0 @@ -/* - * knoxite - * Copyright (c) 2016-2017, Christian Muehlhaeuser - * - * For license see LICENSE - */ - -package knoxite - -import uuid "github.com/nu7hatch/gouuid" - -// A Volume contains various snapshots -// MUST BE encrypted -type Volume struct { - ID string `json:"id"` - Name string `json:"name"` - Description string `json:"description"` - Snapshots []string `json:"snapshots"` -} - -// NewVolume creates a new volume -func NewVolume(name, description string) (*Volume, error) { - vol := Volume{ - Name: name, - Description: description, - } - - u, err := uuid.NewV4() - if err != nil { - return &vol, err - } - vol.ID = u.String()[:8] - - return &vol, nil -} - -// AddSnapshot adds a snapshot to a volume -func (v *Volume) AddSnapshot(id string) error { - v.Snapshots = append(v.Snapshots, id) - return nil -} - -// RemoveSnapshot removes a snapshot from a volume -func (v *Volume) RemoveSnapshot(id string) error { - snapshots := []string{} - found := false - - for _, snapshot := range v.Snapshots { - if snapshot == id { - found = true - } else { - snapshots = append(snapshots, snapshot) - } - } - - if !found { - return ErrSnapshotNotFound - } - - v.Snapshots = snapshots - return nil -} - -// LoadSnapshot loads a snapshot within a volume from a repository -func (v *Volume) LoadSnapshot(id string, repository *Repository) (*Snapshot, error) { - for _, snapshot := range v.Snapshots { - if snapshot == id { - snapshot, err := openSnapshot(id, repository) - return snapshot, err - } - } - - return &Snapshot{}, ErrSnapshotNotFound -} diff --git a/lib/pipeline.go b/pipeline.go similarity index 100% rename from lib/pipeline.go rename to pipeline.go diff --git a/lib/progress.go b/progress.go similarity index 100% rename from lib/progress.go rename to progress.go diff --git a/lib/progress_test.go b/progress_test.go similarity index 100% rename from lib/progress_test.go rename to progress_test.go diff --git a/lib/redundancy.go b/redundancy.go similarity index 100% rename from lib/redundancy.go rename to redundancy.go diff --git a/repository.go b/repository.go index a55a564e..102d67d0 100644 --- a/repository.go +++ b/repository.go @@ -1,267 +1,188 @@ /* * knoxite - * Copyright (c) 2016-2017, Christian Muehlhaeuser + * Copyright (c) 2016-2018, Christian Muehlhaeuser * * For license see LICENSE */ -package main +package knoxite import ( - "encoding/json" "errors" - "fmt" - "io" - "strings" - "os" - "syscall" +) + +// A Repository is a collection of backup snapshots +// MUST BE encrypted +type Repository struct { + Version uint `json:"version"` + Volumes []*Volume `json:"volumes"` + Paths []string `json:"storage"` + // Owner string `json:"owner"` - "github.com/klauspost/shutdown2" - "github.com/muesli/gotable" - "github.com/spf13/cobra" - "golang.org/x/crypto/ssh/terminal" - "github.com/muesli/crunchy" + backend BackendManager + password string +} - knoxite "github.com/knoxite/knoxite/lib" +// Const declarations +const ( + RepositoryVersion = 3 ) // Error declarations var ( - ErrPasswordMismatch = errors.New("Passwords did not match") - - repoCmd = &cobra.Command{ - Use: "repo", - Short: "manage repository", - Long: `The repo command manages repositories`, - RunE: nil, - } - repoInitCmd = &cobra.Command{ - Use: "init", - Short: "initialize a new repository", - Long: `The init command initializes a new repository`, - RunE: func(cmd *cobra.Command, args []string) error { - return executeRepoInit() - }, - } - repoCatCmd = &cobra.Command{ - Use: "cat", - Short: "display repository information as JSON", - Long: `The cat command displays the internal repository information as JSON`, - RunE: func(cmd *cobra.Command, args []string) error { - return executeRepoCat() - }, - } - repoInfoCmd = &cobra.Command{ - Use: "info", - Short: "display repository information", - Long: `The info command displays the repository status & information`, - RunE: func(cmd *cobra.Command, args []string) error { - return executeRepoInfo() - }, - } - repoAddCmd = &cobra.Command{ - Use: "add ", - Short: "add another storage backend to a repository", - Long: `The add command adds another storage backend to a repository`, - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) != 1 { - return fmt.Errorf("add needs a URL to be added") - } - return executeRepoAdd(args[0]) - }, - } - repoPackCmd = &cobra.Command{ - Use: "pack", - Short: "pack repository and release redundant data", - Long: `The pack command deletes all unused data chunks from storage`, - RunE: func(cmd *cobra.Command, args []string) error { - return executeRepoPack() - }, - } + ErrRepositoryIncompatible = errors.New("The repository is not compatible with this version of Knoxite") + ErrOpenRepositoryFailed = errors.New("Wrong password or corrupted repository") + ErrVolumeNotFound = errors.New("Volume not found") + ErrSnapshotNotFound = errors.New("Snapshot not found") ) -func init() { - repoCmd.AddCommand(repoInitCmd) - repoCmd.AddCommand(repoCatCmd) - repoCmd.AddCommand(repoInfoCmd) - repoCmd.AddCommand(repoAddCmd) - repoCmd.AddCommand(repoPackCmd) - RootCmd.AddCommand(repoCmd) -} - -func executeRepoInit() error { - // acquire a shutdown lock. we don't want these next calls to be interrupted - lock := shutdown.Lock() - if lock == nil { - return nil +// NewRepository returns a new repository +func NewRepository(path, password string) (Repository, error) { + repository := Repository{ + Version: RepositoryVersion, + password: password, } - defer lock() - r, err := newRepository(globalOpts.Repo, globalOpts.Password) + backend, err := BackendFromURL(path) if err != nil { - return fmt.Errorf("Creating repository at %s failed: %v", globalOpts.Repo, err) + return repository, err } + repository.backend.AddBackend(&backend) - fmt.Printf("Created new repository at %s\n", (*r.BackendManager().Backends[0]).Location()) - return nil + err = repository.init() + return repository, err } -func executeRepoAdd(url string) error { - // acquire a shutdown lock. we don't want these next calls to be interrupted - lock := shutdown.Lock() - if lock == nil { - return nil +// OpenRepository opens an existing repository +func OpenRepository(path, password string) (Repository, error) { + repository := Repository{ + password: password, } - defer lock() - r, err := openRepository(globalOpts.Repo, globalOpts.Password) + backend, err := BackendFromURL(path) if err != nil { - return err + return repository, err } - - backend, err := knoxite.BackendFromURL(url) + b, err := backend.LoadRepository() if err != nil { - return err + return repository, err } - r.BackendManager().AddBackend(&backend) - err = r.Save() + pipe, err := NewDecodingPipeline(CompressionNone, EncryptionAES, password) if err != nil { - return err - } - fmt.Printf("Added %s to repository\n", backend.Location()) - return nil -} - -func executeRepoCat() error { - r, err := openRepository(globalOpts.Repo, globalOpts.Password) - if err != nil { - return err - } - - json, err := json.MarshalIndent(r, "", " ") - if err != nil { - return err + return repository, err } - fmt.Printf("%s\n", json) - return nil -} - -func executeRepoPack() error { - r, err := openRepository(globalOpts.Repo, globalOpts.Password) + err = pipe.Decode(b, &repository) if err != nil { - return err + return repository, ErrOpenRepositoryFailed } - index, err := knoxite.OpenChunkIndex(&r) - if err != nil { - return err + if repository.Version < RepositoryVersion { + return repository, ErrRepositoryIncompatible } - freedSize, err := index.Pack(&r) - if err != nil { - return err + for _, url := range repository.Paths { + backend, berr := BackendFromURL(url) + if berr != nil { + return repository, berr + } + repository.backend.AddBackend(&backend) } - err = index.Save(&r) - if err != nil { - return err - } + return repository, err +} - fmt.Printf("Freed storage space: %s\n", knoxite.SizeToString(freedSize)) +// AddVolume adds a volume to a repository +func (r *Repository) AddVolume(volume *Volume) error { + r.Volumes = append(r.Volumes, volume) return nil } -func executeRepoInfo() error { - r, err := openRepository(globalOpts.Repo, globalOpts.Password) - if err != nil { - return err +// FindVolume finds a volume within a repository +func (r *Repository) FindVolume(id string) (*Volume, error) { + if id == "latest" && len(r.Volumes) > 0 { + return r.Volumes[len(r.Volumes)-1], nil } - tab := gotable.NewTable([]string{"Storage URL", "Available Space"}, - []int64{-48, 15}, - "No backends found.") - - for _, be := range r.BackendManager().Backends { - space, _ := (*be).AvailableSpace() - tab.AppendRow([]interface{}{ - (*be).Location(), - knoxite.SizeToString(space)}) + for _, volume := range r.Volumes { + if volume.ID == id { + return volume, nil + } } - _ = tab.Print() - return nil + return &Volume{}, ErrVolumeNotFound } -func openRepository(path, password string) (knoxite.Repository, error) { - if password == "" { - var err error - password, err = readPassword("Enter password:") - if err != nil { - return knoxite.Repository{}, err +// FindSnapshot finds a snapshot within a repository +func (r *Repository) FindSnapshot(id string) (*Volume, *Snapshot, error) { + if id == "latest" { + latestVolume := &Volume{} + latestSnapshot := &Snapshot{} + found := false + for _, volume := range r.Volumes { + for _, snapshotID := range volume.Snapshots { + snapshot, err := volume.LoadSnapshot(snapshotID, r) + if err == nil { + if !found || snapshot.Date.Sub(latestSnapshot.Date) > 0 { + latestSnapshot = snapshot + latestVolume = volume + found = true + } + } + } } - } - - return knoxite.OpenRepository(path, password) -} -func newRepository(path, password string) (knoxite.Repository, error) { - if password == "" { - var err error - password, err = readPasswordTwice("Enter a password to encrypt this repository with:", "Confirm password:") - if err != nil { - return knoxite.Repository{}, err + if found { + return latestVolume, latestSnapshot, nil + } + } else { + for _, volume := range r.Volumes { + snapshot, err := volume.LoadSnapshot(id, r) + if err == nil { + return volume, snapshot, err + } } } - return knoxite.NewRepository(path, password) + return &Volume{}, &Snapshot{}, ErrSnapshotNotFound } -func readPassword(prompt string) (string, error) { - var tty io.WriteCloser - tty, err := os.OpenFile("/dev/tty", os.O_WRONLY, 0) - if err != nil { - tty = os.Stdout - } else { - defer tty.Close() +// IsEmpty returns true if there a no snapshots stored in a repository +func (r *Repository) IsEmpty() bool { + for _, volume := range r.Volumes { + if len(volume.Snapshots) > 0 { + return false + } } - fmt.Fprint(tty, prompt+" ") - buf, err := terminal.ReadPassword(int(syscall.Stdin)) - fmt.Fprintln(tty) + return true +} - return string(buf), err +// BackendManager returns the repository's BackendManager +func (r *Repository) BackendManager() *BackendManager { + return &r.backend } -func readPasswordTwice(prompt, promptConfirm string) (string, error) { - pw, err := readPassword(prompt) +// Init creates a new repository +func (r *Repository) init() error { + err := r.backend.InitRepository() if err != nil { - return pw, err + return err } - crunchErr := crunchy.NewValidator().Check(pw) - if crunchErr != nil { - fmt.Printf("Password is considered unsafe: %v\n", crunchErr) - fmt.Printf("Are you sure you want to use this password (y/N)?: ") - var buf string - _, err = fmt.Scan(&buf) - if err != nil { - return pw, err - } + return r.Save() +} - buf = strings.TrimSpace(buf) - buf = strings.ToLower(buf) - if buf != "y" { - return pw, crunchErr - } - } +// Save writes a repository's metadata +func (r *Repository) Save() error { + r.Paths = r.backend.Locations() - pwconfirm, err := readPassword(promptConfirm) + pipe, err := NewEncodingPipeline(CompressionNone, EncryptionAES, r.password) if err != nil { - return pw, err + return err } - if pw != pwconfirm { - return pw, ErrPasswordMismatch + b, err := pipe.Encode(r) + if err != nil { + return err } - - return pw, nil + return r.backend.SaveRepository(b) } diff --git a/lib/repository_test.go b/repository_test.go similarity index 100% rename from lib/repository_test.go rename to repository_test.go diff --git a/lib/scanner.go b/scanner.go similarity index 100% rename from lib/scanner.go rename to scanner.go diff --git a/snapshot.go b/snapshot.go index 4a6c082a..fb004af6 100644 --- a/snapshot.go +++ b/snapshot.go @@ -1,126 +1,227 @@ /* * knoxite - * Copyright (c) 2016-2017, Christian Muehlhaeuser + * Copyright (c) 2016-2018, Christian Muehlhaeuser * * For license see LICENSE */ -package main +package knoxite import ( - "fmt" + "math" + "os" + "path/filepath" + "strings" + "sync" + "time" + + uuid "github.com/nu7hatch/gouuid" +) - "github.com/muesli/gotable" - "github.com/spf13/cobra" +// A Snapshot is a compilation of one or many archives +// MUST BE encrypted +type Snapshot struct { + mut sync.Mutex - knoxite "github.com/knoxite/knoxite/lib" -) + ID string `json:"id"` + Date time.Time `json:"date"` + Description string `json:"description"` + Stats Stats `json:"stats"` + Archives map[string]*Archive `json:"items"` +} -var ( - snapshotCmd = &cobra.Command{ - Use: "snapshot", - Short: "manage snapshots", - Long: `The snapshot command manages snapshots`, - RunE: nil, +// NewSnapshot creates a new snapshot +func NewSnapshot(description string) (*Snapshot, error) { + snapshot := Snapshot{ + Date: time.Now(), + Description: description, + Archives: make(map[string]*Archive), } - snapshotListCmd = &cobra.Command{ - Use: "list ", - Short: "list all snapshots inside a volume", - Long: `The list command lists all snapshots stored in a volume`, - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) != 1 { - return fmt.Errorf("list needs a volume ID to work on") - } - return executeSnapshotList(args[0]) - }, + + u, err := uuid.NewV4() + if err != nil { + return &snapshot, err } - snapshotRemoveCmd = &cobra.Command{ - Use: "remove ", - Short: "remove a snapshot", - Long: `The remove command deletes a snapshot from a volume`, - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) != 1 { - return fmt.Errorf("remove needs a snapshot ID to work on") + snapshot.ID = u.String()[:8] + + return &snapshot, nil +} + +func (snapshot *Snapshot) gatherTargetInformation(cwd string, paths []string, excludes []string, out chan ArchiveResult) { + var wg sync.WaitGroup + for _, path := range paths { + c := findFiles(path, excludes) + + for result := range c { + if result.Error == nil { + rel, err := filepath.Rel(cwd, result.Archive.Path) + if err == nil && !strings.HasPrefix(rel, "../") { + result.Archive.Path = rel + } + if isSpecialPath(result.Archive.Path) { + continue + } + + snapshot.mut.Lock() + snapshot.Stats.Size += result.Archive.Size + switch result.Archive.Type { + case Directory: + snapshot.Stats.Dirs++ + case File: + snapshot.Stats.Files++ + case SymLink: + snapshot.Stats.SymLinks++ + } + snapshot.mut.Unlock() } - return executeSnapshotRemove(args[0]) - }, + + wg.Add(1) + go func(r ArchiveResult) { + out <- r + wg.Done() + }(result) + } } -) -func init() { - snapshotCmd.AddCommand(snapshotListCmd) - snapshotCmd.AddCommand(snapshotRemoveCmd) - RootCmd.AddCommand(snapshotCmd) + wg.Wait() + close(out) } -func executeSnapshotRemove(snapshotID string) error { - repository, err := openRepository(globalOpts.Repo, globalOpts.Password) - if err != nil { - return err - } - chunkIndex, err := knoxite.OpenChunkIndex(&repository) - if err != nil { - return err - } +// Add adds a path to a Snapshot +func (snapshot *Snapshot) Add(cwd string, paths []string, excludes []string, repository Repository, chunkIndex *ChunkIndex, compress, encrypt uint16, dataParts, parityParts uint) chan Progress { + progress := make(chan Progress) + fwd := make(chan ArchiveResult) - volume, snapshot, err := repository.FindSnapshot(snapshotID) - if err != nil { - return err - } + go snapshot.gatherTargetInformation(cwd, paths, excludes, fwd) - err = volume.RemoveSnapshot(snapshot.ID) + go func() { + for result := range fwd { + if result.Error != nil { + p := newProgressError(result.Error) + progress <- p + break + } + + archive := result.Archive + rel, err := filepath.Rel(cwd, archive.Path) + if err == nil && !strings.HasPrefix(rel, "../") { + archive.Path = rel + } + if isSpecialPath(archive.Path) { + continue + } + + p := newProgress(archive) + snapshot.mut.Lock() + p.TotalStatistics = snapshot.Stats + snapshot.mut.Unlock() + progress <- p + + if archive.Type == File { + dataParts = uint(math.Max(1, float64(dataParts))) + chunkchan, err := chunkFile(archive.Path, compress, encrypt, repository.password, int(dataParts), int(parityParts)) + if err != nil { + if os.IsNotExist(err) { + // if this file has already been deleted before we could backup it, we can gracefully ignore it and continue + continue + } + p = newProgressError(err) + progress <- p + break + } + archive.Encrypted = encrypt + archive.Compressed = compress + + for cd := range chunkchan { + if cd.Error != nil { + p = newProgressError(err) + progress <- p + close(progress) + return + } + chunk := cd.Chunk + // fmt.Printf("\tSplit %s (#%d, %d bytes), compression: %s, encryption: %s, hash: %s\n", id.Path, cd.Num, cd.Size, CompressionText(cd.Compressed), EncryptionText(cd.Encrypted), cd.Hash) + + // store this chunk + n, err := repository.backend.StoreChunk(chunk) + if err != nil { + p = newProgressError(err) + progress <- p + close(progress) + return + } + + // release the memory, we don't need the data anymore + chunk.Data = &[][]byte{} + + archive.Chunks = append(archive.Chunks, chunk) + archive.StorageSize += n + + p.CurrentItemStats.StorageSize = archive.StorageSize + p.CurrentItemStats.Transferred += uint64(chunk.OriginalSize) + snapshot.Stats.Transferred += uint64(chunk.OriginalSize) + snapshot.Stats.StorageSize += n + + snapshot.mut.Lock() + p.TotalStatistics = snapshot.Stats + snapshot.mut.Unlock() + progress <- p + } + } + + snapshot.AddArchive(archive) + chunkIndex.AddArchive(archive, snapshot.ID) + } + close(progress) + }() + + return progress +} + +// Clone clones a snapshot +func (snapshot *Snapshot) Clone() (*Snapshot, error) { + s, err := NewSnapshot(snapshot.Description) if err != nil { - return err + return s, err } - chunkIndex.RemoveSnapshot(snapshot.ID) - err = chunkIndex.Save(&repository) + s.Stats = snapshot.Stats + s.Archives = snapshot.Archives + + return s, nil +} + +// openSnapshot opens an existing snapshot +func openSnapshot(id string, repository *Repository) (*Snapshot, error) { + snapshot := Snapshot{ + Archives: make(map[string]*Archive), + } + b, err := repository.backend.LoadSnapshot(id) if err != nil { - return err + return &snapshot, err } - - err = repository.Save() + pipe, err := NewDecodingPipeline(CompressionLZMA, EncryptionAES, repository.password) if err != nil { - return err + return &snapshot, err } - - fmt.Printf("Snapshot %s removed: %s\n", snapshot.ID, snapshot.Stats.String()) - fmt.Println("Do not forget to run 'repo pack' to delete un-referenced chunks and free up storage space!") - return nil + err = pipe.Decode(b, &snapshot) + return &snapshot, err } -func executeSnapshotList(volID string) error { - repository, err := openRepository(globalOpts.Repo, globalOpts.Password) +// Save writes a snapshot's metadata +func (snapshot *Snapshot) Save(repository *Repository) error { + pipe, err := NewEncodingPipeline(CompressionLZMA, EncryptionAES, repository.password) if err != nil { return err } - - volume, err := repository.FindVolume(volID) + b, err := pipe.Encode(snapshot) if err != nil { return err } + return repository.backend.SaveSnapshot(snapshot.ID, b) +} - tab := gotable.NewTable([]string{"ID", "Date", "Original Size", "Storage Size", "Description"}, - []int64{-8, -19, 13, 12, -48}, "No snapshots found. This volume is empty.") - totalSize := uint64(0) - totalStorageSize := uint64(0) - - for _, snapshotID := range volume.Snapshots { - snapshot, err := volume.LoadSnapshot(snapshotID, &repository) - if err != nil { - return err - } - tab.AppendRow([]interface{}{ - snapshot.ID, - snapshot.Date.Format(timeFormat), - knoxite.SizeToString(snapshot.Stats.Size), - knoxite.SizeToString(snapshot.Stats.StorageSize), - snapshot.Description}) - totalSize += snapshot.Stats.Size - totalStorageSize += snapshot.Stats.StorageSize - } - - tab.SetSummary([]interface{}{"", "", knoxite.SizeToString(totalSize), knoxite.SizeToString(totalStorageSize), ""}) - _ = tab.Print() - return nil +// AddArchive adds an archive to a snapshot +func (snapshot *Snapshot) AddArchive(archive *Archive) { + snapshot.Archives[archive.Path] = archive } diff --git a/lib/snapshot_test.go b/snapshot_test.go similarity index 100% rename from lib/snapshot_test.go rename to snapshot_test.go diff --git a/lib/stat_generic.go b/stat_generic.go similarity index 100% rename from lib/stat_generic.go rename to stat_generic.go diff --git a/lib/stat_unix.go b/stat_unix.go similarity index 100% rename from lib/stat_unix.go rename to stat_unix.go diff --git a/lib/stat_windows.go b/stat_windows.go similarity index 100% rename from lib/stat_windows.go rename to stat_windows.go diff --git a/lib/statistics.go b/statistics.go similarity index 100% rename from lib/statistics.go rename to statistics.go diff --git a/lib/statistics_test.go b/statistics_test.go similarity index 100% rename from lib/statistics_test.go rename to statistics_test.go diff --git a/storage/azure/azure_file.go b/storage/azure/azure_file.go index d18845f0..9ff2cbca 100644 --- a/storage/azure/azure_file.go +++ b/storage/azure/azure_file.go @@ -16,7 +16,7 @@ import ( "github.com/Azure/azure-storage-file-go/azfile" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) // AzureFileStorage stores data on an Azure File Storage diff --git a/storage/backblaze/backblaze.go b/storage/backblaze/backblaze.go index 54bc571e..205555a3 100644 --- a/storage/backblaze/backblaze.go +++ b/storage/backblaze/backblaze.go @@ -1,6 +1,6 @@ /* * knoxite - * Copyright (c) 2016-2017, Christian Muehlhaeuser + * Copyright (c) 2016-2020, Christian Muehlhaeuser * Copyright (c) 2016, Nicolas Martin * * For license see LICENSE @@ -17,7 +17,7 @@ import ( "gopkg.in/kothar/go-backblaze.v0" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) // BackblazeStorage stores data on a remote Backblaze diff --git a/storage/backendtester.go b/storage/backendtester.go index 2a788d19..9efab5d5 100644 --- a/storage/backendtester.go +++ b/storage/backendtester.go @@ -18,7 +18,7 @@ import ( "reflect" "testing" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) type BackendTest struct { diff --git a/storage/dropbox/dropbox.go b/storage/dropbox/dropbox.go index d4de4c7b..cfc93be0 100644 --- a/storage/dropbox/dropbox.go +++ b/storage/dropbox/dropbox.go @@ -1,6 +1,6 @@ /* * knoxite - * Copyright (c) 2016-2017, Christian Muehlhaeuser + * Copyright (c) 2016-2020, Christian Muehlhaeuser * Copyright (c) 2016, Nicolas Martin * * For license see LICENSE @@ -16,7 +16,7 @@ import ( "github.com/tj/go-dropbox" "github.com/tj/go-dropy" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) // DropboxStorage stores data on a remote Dropbox diff --git a/storage/ftp/ftp.go b/storage/ftp/ftp.go index 953988e2..bb9f9ef0 100644 --- a/storage/ftp/ftp.go +++ b/storage/ftp/ftp.go @@ -1,6 +1,6 @@ /* * knoxite - * Copyright (c) 2016-2017, Christian Muehlhaeuser + * Copyright (c) 2016-2020, Christian Muehlhaeuser * Copyright (c) 2016, Nicolas Martin * * For license see LICENSE @@ -21,7 +21,7 @@ import ( "github.com/jlaffaye/ftp" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) // FTPStorage stores data on a remote FTP diff --git a/storage/googlecloud/gcloud.go b/storage/googlecloud/gcloud.go index 8c56d6b3..29bab99c 100644 --- a/storage/googlecloud/gcloud.go +++ b/storage/googlecloud/gcloud.go @@ -1,6 +1,7 @@ /* * knoxite * Copyright (c) 2020, Matthias Hartmann + * 2020, Christian Muehlhaeuser * * For license see LICENSE */ @@ -18,7 +19,7 @@ import ( "cloud.google.com/go/storage" "google.golang.org/api/option" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) // GoogleCloudStorage stores data in a Google Cloud Storage bucket diff --git a/storage/googledrive/gdrive.go b/storage/googledrive/gdrive.go index 1e926106..2a24af38 100644 --- a/storage/googledrive/gdrive.go +++ b/storage/googledrive/gdrive.go @@ -1,6 +1,6 @@ /* * knoxite - * Copyright (c) 2016-2017, Christian Muehlhaeuser + * Copyright (c) 2016-2020, Christian Muehlhaeuser * Copyright (c) 2016, Nicolas Martin * * For license see LICENSE @@ -11,7 +11,7 @@ package googledrive import ( "net/url" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) // GoogleDriveStorage stores data on a remote Google Drive diff --git a/storage/http/http.go b/storage/http/http.go index e2c5bf7d..573c8014 100644 --- a/storage/http/http.go +++ b/storage/http/http.go @@ -1,6 +1,6 @@ /* * knoxite - * Copyright (c) 2016-2017, Christian Muehlhaeuser + * Copyright (c) 2016-2020, Christian Muehlhaeuser * * For license see LICENSE */ @@ -17,7 +17,7 @@ import ( "net/url" "strconv" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) // HTTPStorage stores data on a remote HTTP server diff --git a/storage/mega/mega.go b/storage/mega/mega.go index 24597769..960c2a2e 100644 --- a/storage/mega/mega.go +++ b/storage/mega/mega.go @@ -17,7 +17,7 @@ import ( "github.com/t3rm1n4l/go-mega" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) // MegaStorage stores data on a remote Mega diff --git a/storage/s3/s3.go b/storage/s3/s3.go index 3bc11778..8bde2ad0 100644 --- a/storage/s3/s3.go +++ b/storage/s3/s3.go @@ -19,7 +19,7 @@ import ( "github.com/minio/minio-go" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) // S3Storage stores data on a remote AmazonS3 diff --git a/storage/sftp/sftp.go b/storage/sftp/sftp.go index 5046c0f7..e0bb38cb 100644 --- a/storage/sftp/sftp.go +++ b/storage/sftp/sftp.go @@ -19,10 +19,10 @@ import ( "github.com/pkg/sftp" "golang.org/x/crypto/ssh" - agent "golang.org/x/crypto/ssh/agent" + "golang.org/x/crypto/ssh/agent" kh "golang.org/x/crypto/ssh/knownhosts" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) type SFTPStorage struct { diff --git a/storage/webdav/webdav.go b/storage/webdav/webdav.go index e10efa64..93d4136d 100644 --- a/storage/webdav/webdav.go +++ b/storage/webdav/webdav.go @@ -14,7 +14,7 @@ import ( "github.com/studio-b12/gowebdav" - knoxite "github.com/knoxite/knoxite/lib" + "github.com/knoxite/knoxite" ) // WebDAVStorage stores data on a WebDav Server diff --git a/lib/storage_filesystem.go b/storage_filesystem.go similarity index 100% rename from lib/storage_filesystem.go rename to storage_filesystem.go diff --git a/lib/storage_local.go b/storage_local.go similarity index 100% rename from lib/storage_local.go rename to storage_local.go diff --git a/lib/storage_local_unix.go b/storage_local_unix.go similarity index 100% rename from lib/storage_local_unix.go rename to storage_local_unix.go diff --git a/lib/storage_local_windows.go b/storage_local_windows.go similarity index 100% rename from lib/storage_local_windows.go rename to storage_local_windows.go diff --git a/volume.go b/volume.go index 3161ca39..f6ba30b3 100644 --- a/volume.go +++ b/volume.go @@ -5,101 +5,70 @@ * For license see LICENSE */ -package main +package knoxite -import ( - "fmt" +import uuid "github.com/nu7hatch/gouuid" - "github.com/klauspost/shutdown2" - "github.com/muesli/gotable" - "github.com/spf13/cobra" - - knoxite "github.com/knoxite/knoxite/lib" -) - -// VolumeInitOptions holds all the options that can be set for the 'volume init' command -type VolumeInitOptions struct { - Description string +// A Volume contains various snapshots +// MUST BE encrypted +type Volume struct { + ID string `json:"id"` + Name string `json:"name"` + Description string `json:"description"` + Snapshots []string `json:"snapshots"` } -var ( - volumeInitOpts = VolumeInitOptions{} - - volumeCmd = &cobra.Command{ - Use: "volume", - Short: "manage volumes", - Long: `The volume command manages volumes`, - RunE: nil, - } - volumeInitCmd = &cobra.Command{ - Use: "init ", - Short: "initialize a new volume", - Long: `The init command initializes a new volume`, - RunE: func(cmd *cobra.Command, args []string) error { - if len(args) != 1 { - return fmt.Errorf("init needs a name for the new volume") - } - return executeVolumeInit(args[0], volumeInitOpts.Description) - }, - } - volumeListCmd = &cobra.Command{ - Use: "list", - Short: "list all volumes inside a repository", - Long: `The list command lists all volumes stored in a repository`, - RunE: func(cmd *cobra.Command, args []string) error { - return executeVolumeList() - }, +// NewVolume creates a new volume +func NewVolume(name, description string) (*Volume, error) { + vol := Volume{ + Name: name, + Description: description, } -) -func init() { - volumeInitCmd.Flags().StringVarP(&volumeInitOpts.Description, "desc", "d", "", "a description or comment for this volume") + u, err := uuid.NewV4() + if err != nil { + return &vol, err + } + vol.ID = u.String()[:8] - volumeCmd.AddCommand(volumeInitCmd) - volumeCmd.AddCommand(volumeListCmd) - RootCmd.AddCommand(volumeCmd) + return &vol, nil } -func executeVolumeInit(name, description string) error { - // acquire a shutdown lock. we don't want these next calls to be interrupted - lock := shutdown.Lock() - if lock == nil { - return nil - } - defer lock() +// AddSnapshot adds a snapshot to a volume +func (v *Volume) AddSnapshot(id string) error { + v.Snapshots = append(v.Snapshots, id) + return nil +} - repository, err := openRepository(globalOpts.Repo, globalOpts.Password) - if err == nil { - vol, verr := knoxite.NewVolume(name, description) - if verr == nil { - verr = repository.AddVolume(vol) - if verr != nil { - return fmt.Errorf("Creating volume %s failed: %v", name, verr) - } +// RemoveSnapshot removes a snapshot from a volume +func (v *Volume) RemoveSnapshot(id string) error { + snapshots := []string{} + found := false - annotation := "Name: " + vol.Name - if len(vol.Description) > 0 { - annotation += ", Description: " + vol.Description - } - fmt.Printf("Volume %s (%s) created\n", vol.ID, annotation) - return repository.Save() + for _, snapshot := range v.Snapshots { + if snapshot == id { + found = true + } else { + snapshots = append(snapshots, snapshot) } } - return err -} -func executeVolumeList() error { - repository, err := openRepository(globalOpts.Repo, globalOpts.Password) - if err != nil { - return err + if !found { + return ErrSnapshotNotFound } - tab := gotable.NewTable([]string{"ID", "Name", "Description"}, - []int64{-8, -32, -48}, "No volumes found. This repository is empty.") - for _, volume := range repository.Volumes { - tab.AppendRow([]interface{}{volume.ID, volume.Name, volume.Description}) + v.Snapshots = snapshots + return nil +} + +// LoadSnapshot loads a snapshot within a volume from a repository +func (v *Volume) LoadSnapshot(id string, repository *Repository) (*Snapshot, error) { + for _, snapshot := range v.Snapshots { + if snapshot == id { + snapshot, err := openSnapshot(id, repository) + return snapshot, err + } } - _ = tab.Print() - return nil + return &Snapshot{}, ErrSnapshotNotFound } diff --git a/lib/volume_test.go b/volume_test.go similarity index 100% rename from lib/volume_test.go rename to volume_test.go