Merge pull request #511 from restic/cleanups

Cleanups and test functions

run_integration_tests.go

@@ -45,9 +45,8 @@ type CIEnvironment interface {
 
 // TravisEnvironment is the environment in which Travis tests run.
 type TravisEnvironment struct {
-	goxArch []string
-	goxOS   []string
-	minio   string
+	goxOSArch []string
+	minio     string
 
 	minioSrv     *Background
 	minioTempdir string
@@ -175,24 +174,27 @@ func (env *TravisEnvironment) Prepare() error {
 			return err
 		}
 		if runtime.GOOS == "linux" {
-			env.goxArch = []string{"386", "amd64"}
+			env.goxOSArch = []string{
+				"linux/386", "linux/amd64",
+				"windows/386", "windows/amd64",
+				"darwin/386", "darwin/amd64",
+				"freebsd/386", "freebsd/amd64",
+				"opendbsd/386", "opendbsd/amd64",
+			}
 			if !strings.HasPrefix(runtime.Version(), "go1.3") {
-				env.goxArch = append(env.goxArch, "arm")
+				env.goxOSArch = append(env.goxOSArch,
+					"linux/arm", "freebsd/arm")
 			}
-
-			env.goxOS = []string{"linux", "darwin", "freebsd", "openbsd", "windows"}
 		} else {
-			env.goxArch = []string{runtime.GOARCH}
-			env.goxOS = []string{runtime.GOOS}
+			env.goxOSArch = []string{runtime.GOOS + "/" + runtime.GOARCH}
 		}
 
-		msg("gox: OS %v, ARCH %v\n", env.goxOS, env.goxArch)
+		msg("gox: OS/ARCH %v\n", env.goxOSArch)
 
 		v := runtime.Version()
 		if !strings.HasPrefix(v, "go1.5") && !strings.HasPrefix(v, "go1.6") {
 			err := run("gox", "-build-toolchain",
-				"-os", strings.Join(env.goxOS, " "),
-				"-arch", strings.Join(env.goxArch, " "))
+				"-osarch", strings.Join(env.goxOSArch, " "))
 
 			if err != nil {
 				return err
@@ -320,8 +322,7 @@ func (env *TravisEnvironment) RunTests() error {
 		// compile for all target architectures with tags
 		for _, tags := range []string{"release", "debug"} {
 			runWithEnv(env.env, "gox", "-verbose",
-				"-os", strings.Join(env.goxOS, " "),
-				"-arch", strings.Join(env.goxArch, " "),
+				"-osarch", strings.Join(env.goxOSArch, " "),
 				"-tags", tags,
 				"-output", "/tmp/{{.Dir}}_{{.OS}}_{{.Arch}}",
 				"cmds/restic")

src/cmds/restic/cmd_rebuild_index.go

@@ -33,17 +33,7 @@ func loadBlobsFromPacks(repo *repository.Repository) (packs map[backend.ID][]pac
 	defer close(done)
 
 	f := func(job worker.Job, done <-chan struct{}) (interface{}, error) {
-		id := job.Data.(backend.ID)
-
-		h := backend.Handle{Type: backend.Data, Name: id.String()}
-		rd := backend.NewReadSeeker(repo.Backend(), h)
-
-		unpacker, err := pack.NewUnpacker(repo.Key(), rd)
-		if err != nil {
-			return nil, err
-		}
-
-		return unpacker.Entries, nil
+		return repo.ListPack(job.Data.(backend.ID))
 	}
 
 	jobCh := make(chan worker.Job)

src/restic/archiver.go

@@ -1,7 +1,6 @@
 package restic
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"io"
@@ -92,15 +91,15 @@ func (arch *Archiver) isKnownBlob(id backend.ID) bool {
 }
 
 // Save stores a blob read from rd in the repository.
-func (arch *Archiver) Save(t pack.BlobType, id backend.ID, length uint, rd io.Reader) error {
+func (arch *Archiver) Save(t pack.BlobType, data []byte, id backend.ID) error {
 	debug.Log("Archiver.Save", "Save(%v, %v)\n", t, id.Str())
 
 	if arch.isKnownBlob(id) {
 		debug.Log("Archiver.Save", "blob %v is known\n", id.Str())
 		return nil
 	}
 
-	err := arch.repo.SaveFrom(t, &id, length, rd)
+	_, err := arch.repo.SaveAndEncrypt(t, data, &id)
 	if err != nil {
 		debug.Log("Archiver.Save", "Save(%v, %v): error %v\n", t, id.Str(), err)
 		return err
@@ -160,7 +159,7 @@ func (arch *Archiver) saveChunk(chunk chunker.Chunk, p *Progress, token struct{}
 	defer freeBuf(chunk.Data)
 
 	id := backend.Hash(chunk.Data)
-	err := arch.Save(pack.Data, id, chunk.Length, bytes.NewReader(chunk.Data))
+	err := arch.Save(pack.Data, chunk.Data, id)
 	// TODO handle error
 	if err != nil {
 		panic(err)

src/restic/archiver_duplication_test.go

@@ -1,7 +1,6 @@
 package restic_test
 
 import (
-	"bytes"
 	"crypto/rand"
 	"errors"
 	"io"
@@ -108,7 +107,7 @@ func testArchiverDuplication(t *testing.T) {
 
 				buf := make([]byte, 50)
 
-				err := arch.Save(pack.Data, id, uint(len(buf)), bytes.NewReader(buf))
+				err := arch.Save(pack.Data, buf, id)
 				if err != nil {
 					t.Fatal(err)
 				}

src/restic/archiver_test.go

@@ -267,7 +267,7 @@ func testParallelSaveWithDuplication(t *testing.T, seed int) {
 
 				id := backend.Hash(c.Data)
 				time.Sleep(time.Duration(id[0]))
-				err := arch.Save(pack.Data, id, c.Length, bytes.NewReader(c.Data))
+				err := arch.Save(pack.Data, c.Data, id)
 				<-barrier
 				errChan <- err
 			}(c, errChan)

src/restic/checker/checker.go

@@ -534,6 +534,7 @@ func filterTrees(backlog backend.IDs, loaderChan chan<- backend.ID, in <-chan tr
 			inCh = nil
 
 		case outCh <- job:
+			debug.Log("checker.FilterTrees", "tree sent to check: %v", job.ID.Str())
 			outCh = nil
 			inCh = in
 		}
@@ -581,6 +582,14 @@ func (c *Checker) checkTree(id backend.ID, tree *restic.Tree) (errs []error) {
 	for _, node := range tree.Nodes {
 		switch node.Type {
 		case "file":
+			if node.Content == nil {
+				errs = append(errs, Error{TreeID: id, Err: fmt.Errorf("file %q has nil blob list", node.Name)})
+			}
+
+			if node.Mode == 0 {
+				errs = append(errs, Error{TreeID: id, Err: fmt.Errorf("file %q has invalid mode: %v", node.Name, node.Mode)})
+			}
+
 			for b, blobID := range node.Content {
 				if blobID.IsNull() {
 					errs = append(errs, Error{TreeID: id, Err: fmt.Errorf("file %q blob %d has null ID", node.Name, b)})
@@ -598,6 +607,16 @@ func (c *Checker) checkTree(id backend.ID, tree *restic.Tree) (errs []error) {
 				errs = append(errs, Error{TreeID: id, Err: fmt.Errorf("dir node %q subtree id is null", node.Name)})
 				continue
 			}
+
+		case "symlink":
+			// nothing to check
+
+		default:
+			errs = append(errs, Error{TreeID: id, Err: fmt.Errorf("node %q with invalid type %q", node.Name, node.Type)})
+		}
+
+		if node.Name == "" {
+			errs = append(errs, Error{TreeID: id, Err: errors.New("node with empty name")})
 		}
 	}
 

src/restic/progress.go

@@ -19,7 +19,7 @@ type Progress struct {
 	start      time.Time
 	c          *time.Ticker
 	cancel     chan struct{}
-	o          sync.Once
+	o          *sync.Once
 	d          time.Duration
 	lastUpdate time.Time
 
@@ -52,7 +52,7 @@ func (p *Progress) Start() {
 		return
 	}
 
-	p.o = sync.Once{}
+	p.o = &sync.Once{}
 	p.cancel = make(chan struct{})
 	p.running = true
 	p.Reset()

src/restic/repository/repository.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"io/ioutil"
 	"os"
 
 	"restic/backend"
@@ -212,26 +211,6 @@ func (r *Repository) SaveAndEncrypt(t pack.BlobType, data []byte, id *backend.ID
 	return *id, r.savePacker(packer)
 }
 
-// SaveFrom encrypts data read from rd and stores it in a pack in the backend as type t.
-func (r *Repository) SaveFrom(t pack.BlobType, id *backend.ID, length uint, rd io.Reader) error {
-	debug.Log("Repo.SaveFrom", "save id %v (%v, %d bytes)", id.Str(), t, length)
-	if id == nil {
-		return errors.New("id is nil")
-	}
-
-	buf, err := ioutil.ReadAll(rd)
-	if err != nil {
-		return err
-	}
-
-	_, err = r.SaveAndEncrypt(t, buf, id)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
 // SaveJSON serialises item as JSON and encrypts and saves it in a pack in the
 // backend as type t.
 func (r *Repository) SaveJSON(t pack.BlobType, item interface{}) (backend.ID, error) {
@@ -539,6 +518,19 @@ func (r *Repository) List(t backend.Type, done <-chan struct{}) <-chan backend.I
 	return outCh
 }
 
+// ListPack returns the list of blobs saved in the pack id.
+func (r *Repository) ListPack(id backend.ID) ([]pack.Blob, error) {
+	h := backend.Handle{Type: backend.Data, Name: id.String()}
+	rd := backend.NewReadSeeker(r.Backend(), h)
+
+	unpacker, err := pack.NewUnpacker(r.Key(), rd)
+	if err != nil {
+		return nil, err
+	}
+
+	return unpacker.Entries, nil
+}
+
 // Delete calls backend.Delete() if implemented, and returns an error
 // otherwise.
 func (r *Repository) Delete() error {

src/restic/repository/repository_test.go

@@ -117,8 +117,9 @@ func TestSaveFrom(t *testing.T) {
 		id := backend.Hash(data)
 
 		// save
-		err = repo.SaveFrom(pack.Data, &id, uint(size), bytes.NewReader(data))
+		id2, err := repo.SaveAndEncrypt(pack.Data, data, &id)
 		OK(t, err)
+		Equals(t, id, id2)
 
 		OK(t, repo.Flush())
 
@@ -136,7 +137,7 @@ func TestSaveFrom(t *testing.T) {
 	}
 }
 
-func BenchmarkSaveFrom(t *testing.B) {
+func BenchmarkSaveAndEncrypt(t *testing.B) {
 	repo := SetupRepo()
 	defer TeardownRepo(repo)
 
@@ -153,7 +154,7 @@ func BenchmarkSaveFrom(t *testing.B) {
 
 	for i := 0; i < t.N; i++ {
 		// save
-		err = repo.SaveFrom(pack.Data, &id, uint(size), bytes.NewReader(data))
+		_, err = repo.SaveAndEncrypt(pack.Data, data, &id)
 		OK(t, err)
 	}
 }

src/restic/repository/testing.go

@@ -0,0 +1,63 @@
+package repository
+
+import (
+	"os"
+	"restic/backend"
+	"restic/backend/local"
+	"restic/backend/mem"
+	"testing"
+)
+
+// TestBackend returns a fully configured in-memory backend.
+func TestBackend(t testing.TB) (be backend.Backend, cleanup func()) {
+	return mem.New(), func() {}
+}
+
+// TestPassword is used for all repositories created by the Test* functions.
+const TestPassword = "geheim"
+
+// TestRepositoryWithBackend returns a repository initialized with a test
+// password. If be is nil, an in-memory backend is used.
+func TestRepositoryWithBackend(t testing.TB, be backend.Backend) (r *Repository, cleanup func()) {
+	var beCleanup func()
+	if be == nil {
+		be, beCleanup = TestBackend(t)
+	}
+
+	r = New(be)
+
+	err := r.Init(TestPassword)
+	if err != nil {
+		t.Fatalf("TestRepopository(): initialize repo failed: %v", err)
+	}
+
+	return r, func() {
+		if beCleanup != nil {
+			beCleanup()
+		}
+	}
+}
+
+// TestRepository returns a repository initialized with a test password on an
+// in-memory backend. When the environment variable RESTIC_TEST_REPO is set to
+// a non-existing directory, a local backend is created there and this is used
+// instead. The directory is not removed.
+func TestRepository(t testing.TB) (r *Repository, cleanup func()) {
+	dir := os.Getenv("RESTIC_TEST_REPO")
+	if dir != "" {
+		_, err := os.Stat(dir)
+		if err != nil {
+			be, err := local.Create(dir)
+			if err != nil {
+				t.Fatalf("error creating local backend at %v: %v", dir, err)
+			}
+			return TestRepositoryWithBackend(t, be)
+		}
+
+		if err == nil {
+			t.Logf("directory at %v already exists, using mem backend", dir)
+		}
+	}
+
+	return TestRepositoryWithBackend(t, nil)
+}

src/restic/snapshot.go

@@ -11,6 +11,7 @@ import (
 	"restic/repository"
 )
 
+// Snapshot is the state of a resource at one point in time.
 type Snapshot struct {
 	Time     time.Time   `json:"time"`
 	Parent   *backend.ID `json:"parent,omitempty"`
@@ -25,6 +26,8 @@ type Snapshot struct {
 	id *backend.ID // plaintext ID, used during restore
 }
 
+// NewSnapshot returns an initialized snapshot struct for the current user and
+// time.
 func NewSnapshot(paths []string) (*Snapshot, error) {
 	for i, path := range paths {
 		if p, err := filepath.Abs(path); err != nil {
@@ -50,6 +53,7 @@ func NewSnapshot(paths []string) (*Snapshot, error) {
 	return sn, nil
 }
 
+// LoadSnapshot loads the snapshot with the id and returns it.
 func LoadSnapshot(repo *repository.Repository, id backend.ID) (*Snapshot, error) {
 	sn := &Snapshot{id: &id}
 	err := repo.LoadJSONUnpacked(backend.Snapshot, id, sn)
@@ -60,10 +64,28 @@ func LoadSnapshot(repo *repository.Repository, id backend.ID) (*Snapshot, error)
 	return sn, nil
 }
 
+// LoadAllSnapshots returns a list of all snapshots in the repo.
+func LoadAllSnapshots(repo *repository.Repository) (snapshots []*Snapshot, err error) {
+	done := make(chan struct{})
+	defer close(done)
+
+	for id := range repo.List(backend.Snapshot, done) {
+		sn, err := LoadSnapshot(repo, id)
+		if err != nil {
+			return nil, err
+		}
+
+		snapshots = append(snapshots, sn)
+	}
+
+	return snapshots, nil
+}
+
 func (sn Snapshot) String() string {
 	return fmt.Sprintf("<Snapshot of %v at %s>", sn.Paths, sn.Time)
 }
 
+// ID retuns the snapshot's ID.
 func (sn Snapshot) ID() *backend.ID {
 	return sn.id
 }