227 changes: 156 additions & 71 deletions src/cmds/restic/global.go
View file

Large diffs are not rendered by default.

203 changes: 126 additions & 77 deletions src/cmds/restic/integration_fuse_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
// +build ignore
// +build !openbsd
// +build !windows

Expand All @@ -12,7 +13,6 @@ import (
"time"

"restic"
"restic/backend"
"restic/repository"
. "restic/test"
)
Expand All @@ -23,81 +23,129 @@ const (
mountTestSubdir = "snapshots"
)

func snapshotsDirExists(t testing.TB, dir string) bool {
f, err := os.Open(filepath.Join(dir, mountTestSubdir))
if err != nil && os.IsNotExist(err) {
return false
}

if err != nil {
t.Error(err)
}

if err := f.Close(); err != nil {
t.Error(err)
}

return true
}

// waitForMount blocks (max mountWait * mountSleep) until the subdir
// "snapshots" appears in the dir.
func waitForMount(dir string) error {
func waitForMount(t testing.TB, dir string) {
for i := 0; i < mountWait; i++ {
f, err := os.Open(dir)
if err != nil {
return err
if snapshotsDirExists(t, dir) {
t.Log("mounted directory is ready")
return
}

names, err := f.Readdirnames(-1)
if err != nil {
return err
}
time.Sleep(mountSleep)
}

if err = f.Close(); err != nil {
return err
}
t.Errorf("subdir %q of dir %s never appeared", mountTestSubdir, dir)
}

for _, name := range names {
if name == mountTestSubdir {
return nil
}
func mount(t testing.TB, global GlobalOptions, dir string) {
cmd := &CmdMount{global: &global}
OK(t, cmd.Mount(dir))
}

func umount(t testing.TB, global GlobalOptions, dir string) {
cmd := &CmdMount{global: &global}

var err error
for i := 0; i < mountWait; i++ {
if err = cmd.Umount(dir); err == nil {
t.Logf("directory %v umounted", dir)
return
}

time.Sleep(mountSleep)
}

return fmt.Errorf("subdir %q of dir %s never appeared", mountTestSubdir, dir)
t.Errorf("unable to umount dir %v, last error was: %v", dir, err)
}

func listSnapshots(t testing.TB, dir string) []string {
snapshotsDir, err := os.Open(filepath.Join(dir, "snapshots"))
OK(t, err)
names, err := snapshotsDir.Readdirnames(-1)
OK(t, err)
OK(t, snapshotsDir.Close())
return names
}

func cmdMount(t testing.TB, global GlobalOptions, dir string, ready, done chan struct{}) {
defer func() {
ready <- struct{}{}
}()
func checkSnapshots(t testing.TB, global GlobalOptions, repo *repository.Repository, mountpoint, repodir string, snapshotIDs restic.IDs) {
t.Logf("checking for %d snapshots: %v", len(snapshotIDs), snapshotIDs)
go mount(t, global, mountpoint)
waitForMount(t, mountpoint)
defer umount(t, global, mountpoint)

cmd := &CmdMount{global: &global, ready: ready, done: done}
OK(t, cmd.Execute([]string{dir}))
if TestCleanupTempDirs {
RemoveAll(t, dir)
if !snapshotsDirExists(t, mountpoint) {
t.Fatal(`virtual directory "snapshots" doesn't exist`)
}
}

func TestMount(t *testing.T) {
if !RunFuseTest {
t.Skip("Skipping fuse tests")
ids := listSnapshots(t, repodir)
t.Logf("found %v snapshots in repo: %v", len(ids), ids)

namesInSnapshots := listSnapshots(t, mountpoint)
t.Logf("found %v snapshots in fuse mount: %v", len(namesInSnapshots), namesInSnapshots)
Assert(t,
len(namesInSnapshots) == len(snapshotIDs),
"Invalid number of snapshots: expected %d, got %d", len(snapshotIDs), len(namesInSnapshots))

namesMap := make(map[string]bool)
for _, name := range namesInSnapshots {
namesMap[name] = false
}

checkSnapshots := func(repo *repository.Repository, mountpoint string, snapshotIDs []backend.ID) {
snapshotsDir, err := os.Open(filepath.Join(mountpoint, "snapshots"))
for _, id := range snapshotIDs {
snapshot, err := restic.LoadSnapshot(repo, id)
OK(t, err)
namesInSnapshots, err := snapshotsDir.Readdirnames(-1)
OK(t, err)
Assert(t,
len(namesInSnapshots) == len(snapshotIDs),
"Invalid number of snapshots: expected %d, got %d", len(snapshotIDs), len(namesInSnapshots))

namesMap := make(map[string]bool)
for _, name := range namesInSnapshots {
namesMap[name] = false
ts := snapshot.Time.Format(time.RFC3339)
present, ok := namesMap[ts]
if !ok {
t.Errorf("Snapshot %v (%q) isn't present in fuse dir", id.Str(), ts)
}

for _, id := range snapshotIDs {
snapshot, err := restic.LoadSnapshot(repo, id)
OK(t, err)
_, ok := namesMap[snapshot.Time.Format(time.RFC3339)]
Assert(t, ok, "Snapshot %s isn't present in fuse dir", snapshot.Time.Format(time.RFC3339))
namesMap[snapshot.Time.Format(time.RFC3339)] = true
}
for name, present := range namesMap {
Assert(t, present, "Directory %s is present in fuse dir but is not a snapshot", name)
for i := 1; present; i++ {
ts = fmt.Sprintf("%s-%d", snapshot.Time.Format(time.RFC3339), i)
present, ok = namesMap[ts]
if !ok {
t.Errorf("Snapshot %v (%q) isn't present in fuse dir", id.Str(), ts)
}

if !present {
break
}
}
OK(t, snapshotsDir.Close())

namesMap[ts] = true
}

for name, present := range namesMap {
Assert(t, present, "Directory %s is present in fuse dir but is not a snapshot", name)
}
}

func TestMount(t *testing.T) {
if !RunFuseTest {
t.Skip("Skipping fuse tests")
}

withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {

cmdInit(t, global)
repo, err := global.OpenRepository()
OK(t, err)
Expand All @@ -108,55 +156,56 @@ func TestMount(t *testing.T) {
// We remove the mountpoint now to check that cmdMount creates it
RemoveAll(t, mountpoint)

ready := make(chan struct{}, 2)
done := make(chan struct{})
go cmdMount(t, global, mountpoint, ready, done)
<-ready
defer close(done)
OK(t, waitForMount(mountpoint))

mountpointDir, err := os.Open(mountpoint)
OK(t, err)
names, err := mountpointDir.Readdirnames(-1)
OK(t, err)
Assert(t, len(names) == 1 && names[0] == "snapshots", `The fuse virtual directory "snapshots" doesn't exist`)
OK(t, mountpointDir.Close())

checkSnapshots(repo, mountpoint, []backend.ID{})

datafile := filepath.Join("testdata", "backup-data.tar.gz")
fd, err := os.Open(datafile)
if os.IsNotExist(err) {
t.Skipf("unable to find data file %q, skipping", datafile)
return
}
OK(t, err)
OK(t, fd.Close())
checkSnapshots(t, global, repo, mountpoint, env.repo, []restic.ID{})

SetupTarTestFixture(t, env.testdata, datafile)
SetupTarTestFixture(t, env.testdata, filepath.Join("testdata", "backup-data.tar.gz"))

// first backup
cmdBackup(t, global, []string{env.testdata}, nil)
snapshotIDs := cmdList(t, global, "snapshots")
Assert(t, len(snapshotIDs) == 1,
"expected one snapshot, got %v", snapshotIDs)

checkSnapshots(repo, mountpoint, snapshotIDs)
checkSnapshots(t, global, repo, mountpoint, env.repo, snapshotIDs)

// second backup, implicit incremental
cmdBackup(t, global, []string{env.testdata}, nil)
snapshotIDs = cmdList(t, global, "snapshots")
Assert(t, len(snapshotIDs) == 2,
"expected two snapshots, got %v", snapshotIDs)

checkSnapshots(repo, mountpoint, snapshotIDs)
checkSnapshots(t, global, repo, mountpoint, env.repo, snapshotIDs)

// third backup, explicit incremental
cmdBackup(t, global, []string{env.testdata}, &snapshotIDs[0])
snapshotIDs = cmdList(t, global, "snapshots")
Assert(t, len(snapshotIDs) == 3,
"expected three snapshots, got %v", snapshotIDs)

checkSnapshots(repo, mountpoint, snapshotIDs)
checkSnapshots(t, global, repo, mountpoint, env.repo, snapshotIDs)
})
}

func TestMountSameTimestamps(t *testing.T) {
if !RunFuseTest {
t.Skip("Skipping fuse tests")
}

withTestEnvironment(t, func(env *testEnvironment, global GlobalOptions) {
SetupTarTestFixture(t, env.base, filepath.Join("testdata", "repo-same-timestamps.tar.gz"))

repo, err := global.OpenRepository()
OK(t, err)

mountpoint, err := ioutil.TempDir(TestTempDir, "restic-test-mount-")
OK(t, err)

ids := []restic.ID{
restic.TestParseID("280303689e5027328889a06d718b729e96a1ce6ae9ef8290bff550459ae611ee"),
restic.TestParseID("75ad6cdc0868e082f2596d5ab8705e9f7d87316f5bf5690385eeff8dbe49d9f5"),
restic.TestParseID("5fd0d8b2ef0fa5d23e58f1e460188abb0f525c0f0c4af8365a1280c807a80a1b"),
}

checkSnapshots(t, global, repo, mountpoint, env.repo, ids)
})
}
47 changes: 15 additions & 32 deletions src/cmds/restic/integration_helpers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (
"runtime"
"testing"

"restic/repository"
. "restic/test"
)

Expand Down Expand Up @@ -165,34 +166,15 @@ type testEnvironment struct {
base, cache, repo, testdata string
}

func configureRestic(t testing.TB, cache, repo string) GlobalOptions {
return GlobalOptions{
CacheDir: cache,
Repo: repo,
Quiet: true,

password: TestPassword,
stdout: os.Stdout,
stderr: os.Stderr,
}
}

func cleanupTempdir(t testing.TB, tempdir string) {
if !TestCleanupTempDirs {
t.Logf("leaving temporary directory %v used for test", tempdir)
return
}

RemoveAll(t, tempdir)
}

// withTestEnvironment creates a test environment and calls f with it. After f has
// returned, the temporary directory is removed.
func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions)) {
if !RunIntegrationTest {
t.Skip("integration tests disabled")
}

repository.TestUseLowSecurityKDFParameters(t)

tempdir, err := ioutil.TempDir(TestTempDir, "restic-test-")
OK(t, err)

Expand All @@ -207,7 +189,18 @@ func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions))
OK(t, os.MkdirAll(env.cache, 0700))
OK(t, os.MkdirAll(env.repo, 0700))

f(&env, configureRestic(t, env.cache, env.repo))
gopts := GlobalOptions{
Repo: env.repo,
Quiet: true,
password: TestPassword,
stdout: os.Stdout,
stderr: os.Stderr,
}

// always overwrite global options
globalOptions = gopts

f(&env, gopts)

if !TestCleanupTempDirs {
t.Logf("leaving temporary directory %v used for test", tempdir)
Expand All @@ -216,13 +209,3 @@ func withTestEnvironment(t testing.TB, f func(*testEnvironment, GlobalOptions))

RemoveAll(t, tempdir)
}

// removeFile resets the read-only flag and then deletes the file.
func removeFile(fn string) error {
err := os.Chmod(fn, 0666)
if err != nil {
return err
}

return os.Remove(fn)
}
520 changes: 286 additions & 234 deletions src/cmds/restic/integration_test.go
View file

Large diffs are not rendered by default.

18 changes: 9 additions & 9 deletions src/cmds/restic/lock.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ func lockRepository(repo *repository.Repository, exclusive bool) (*restic.Lock,

globalLocks.Lock()
if globalLocks.cancelRefresh == nil {
debug.Log("main.lockRepository", "start goroutine for lock refresh")
debug.Log("start goroutine for lock refresh")
globalLocks.cancelRefresh = make(chan struct{})
globalLocks.refreshWG = sync.WaitGroup{}
globalLocks.refreshWG.Add(1)
Expand All @@ -55,7 +55,7 @@ func lockRepository(repo *repository.Repository, exclusive bool) (*restic.Lock,
var refreshInterval = 5 * time.Minute

func refreshLocks(wg *sync.WaitGroup, done <-chan struct{}) {
debug.Log("main.refreshLocks", "start")
debug.Log("start")
defer func() {
wg.Done()
globalLocks.Lock()
Expand All @@ -68,10 +68,10 @@ func refreshLocks(wg *sync.WaitGroup, done <-chan struct{}) {
for {
select {
case <-done:
debug.Log("main.refreshLocks", "terminate")
debug.Log("terminate")
return
case <-ticker.C:
debug.Log("main.refreshLocks", "refreshing locks")
debug.Log("refreshing locks")
globalLocks.Lock()
for _, lock := range globalLocks.locks {
err := lock.Refresh()
Expand All @@ -88,9 +88,9 @@ func unlockRepo(lock *restic.Lock) error {
globalLocks.Lock()
defer globalLocks.Unlock()

debug.Log("unlockRepo", "unlocking repository")
debug.Log("unlocking repository")
if err := lock.Unlock(); err != nil {
debug.Log("unlockRepo", "error while unlocking: %v", err)
debug.Log("error while unlocking: %v", err)
return err
}

Expand All @@ -108,13 +108,13 @@ func unlockAll() error {
globalLocks.Lock()
defer globalLocks.Unlock()

debug.Log("unlockAll", "unlocking %d locks", len(globalLocks.locks))
debug.Log("unlocking %d locks", len(globalLocks.locks))
for _, lock := range globalLocks.locks {
if err := lock.Unlock(); err != nil {
debug.Log("unlockAll", "error while unlocking: %v", err)
debug.Log("error while unlocking: %v", err)
return err
}
debug.Log("unlockAll", "successfully removed lock")
debug.Log("successfully removed lock")
}

return nil
Expand Down
54 changes: 33 additions & 21 deletions src/cmds/restic/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,38 +3,50 @@ package main
import (
"fmt"
"os"
"runtime"

"github.com/jessevdk/go-flags"
"restic"
"restic/debug"
"runtime"

"github.com/spf13/cobra"

"restic/errors"
)

// cmdRoot is the base command when no other command has been specified.
var cmdRoot = &cobra.Command{
Use: "restic",
Short: "backup and restore files",
Long: `
restic is a backup program which allows saving multiple revisions of files and
directories in an encrypted repository stored on different backends.
`,
SilenceErrors: true,
SilenceUsage: true,
PersistentPreRun: parseEnvironment,
}

func init() {
// set GOMAXPROCS to number of CPUs
runtime.GOMAXPROCS(runtime.NumCPU())
if runtime.Version() < "go1.5" {
gomaxprocs := os.Getenv("GOMAXPROCS")
debug.Log("read GOMAXPROCS from env variable, value: %s", gomaxprocs)
if gomaxprocs == "" {
runtime.GOMAXPROCS(runtime.NumCPU())
}
}
}

func main() {
// defer profile.Start(profile.MemProfileRate(100000), profile.ProfilePath(".")).Stop()
// defer profile.Start(profile.CPUProfile, profile.ProfilePath(".")).Stop()
globalOpts.Repo = os.Getenv("RESTIC_REPOSITORY")
globalOpts.password = os.Getenv("RESTIC_PASSWORD")

debug.Log("restic", "main %#v", os.Args)

_, err := parser.Parse()
if e, ok := err.(*flags.Error); ok && e.Type == flags.ErrHelp {
parser.WriteHelp(os.Stdout)
os.Exit(0)
}
debug.Log("main %#v", os.Args)
err := cmdRoot.Execute()

if err != nil {
switch {
case restic.IsAlreadyLocked(errors.Cause(err)):
fmt.Fprintf(os.Stderr, "%v\nthe `unlock` command can be used to remove stale locks\n", err)
case errors.IsFatal(errors.Cause(err)):
fmt.Fprintf(os.Stderr, "%v\n", err)
}

if restic.IsAlreadyLocked(err) {
fmt.Fprintf(os.Stderr, "\nthe `unlock` command can be used to remove stale locks\n")
case err != nil:
fmt.Fprintf(os.Stderr, "%+v\n", err)
}

RunCleanupHandlers()
Expand Down
46 changes: 46 additions & 0 deletions src/cmds/restic/table.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
package main

import (
"fmt"
"io"
"strings"
)

// Table contains data for a table to be printed.
type Table struct {
Header string
Rows [][]interface{}

RowFormat string
}

// NewTable initializes a new Table.
func NewTable() Table {
return Table{
Rows: [][]interface{}{},
}
}

// Write prints the table to w.
func (t Table) Write(w io.Writer) error {
_, err := fmt.Fprintln(w, t.Header)
if err != nil {
return err
}
_, err = fmt.Fprintln(w, strings.Repeat("-", 70))
if err != nil {
return err
}

for _, row := range t.Rows {
_, err = fmt.Fprintf(w, t.RowFormat+"\n", row...)
if err != nil {
return err
}
}

return nil
}

// TimeFormat is the format used for all timestamps printed by restic.
const TimeFormat = "2006-01-02 15:04:05"
Binary file added src/cmds/restic/testdata/repo-same-timestamps.tar.gz
View file
Binary file not shown.
122 changes: 0 additions & 122 deletions src/restic/archive_reader.go
View file

This file was deleted.

103 changes: 103 additions & 0 deletions src/restic/archiver/archive_reader.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,103 @@
package archiver

import (
"io"
"restic"
"restic/debug"
"time"

"restic/errors"

"github.com/restic/chunker"
)

// ArchiveReader reads from the reader and archives the data. Returned is the
// resulting snapshot and its ID.
func ArchiveReader(repo restic.Repository, p *restic.Progress, rd io.Reader, name string, tags []string) (*restic.Snapshot, restic.ID, error) {
debug.Log("start archiving %s", name)
sn, err := restic.NewSnapshot([]string{name}, tags)
if err != nil {
return nil, restic.ID{}, err
}

p.Start()
defer p.Done()

chnker := chunker.New(rd, repo.Config().ChunkerPolynomial)

var ids restic.IDs
var fileSize uint64

for {
chunk, err := chnker.Next(getBuf())
if errors.Cause(err) == io.EOF {
break
}

if err != nil {
return nil, restic.ID{}, errors.Wrap(err, "chunker.Next()")
}

id := restic.Hash(chunk.Data)

if !repo.Index().Has(id, restic.DataBlob) {
_, err := repo.SaveBlob(restic.DataBlob, chunk.Data, id)
if err != nil {
return nil, restic.ID{}, err
}
debug.Log("saved blob %v (%d bytes)\n", id.Str(), chunk.Length)
} else {
debug.Log("blob %v already saved in the repo\n", id.Str())
}

freeBuf(chunk.Data)

ids = append(ids, id)

p.Report(restic.Stat{Bytes: uint64(chunk.Length)})
fileSize += uint64(chunk.Length)
}

tree := &restic.Tree{
Nodes: []*restic.Node{
&restic.Node{
Name: name,
AccessTime: time.Now(),
ModTime: time.Now(),
Type: "file",
Mode: 0644,
Size: fileSize,
UID: sn.UID,
GID: sn.GID,
User: sn.Username,
Content: ids,
},
},
}

treeID, err := repo.SaveTree(tree)
if err != nil {
return nil, restic.ID{}, err
}
sn.Tree = &treeID
debug.Log("tree saved as %v", treeID.Str())

id, err := repo.SaveJSONUnpacked(restic.SnapshotFile, sn)
if err != nil {
return nil, restic.ID{}, err
}

debug.Log("snapshot saved as %v", id.Str())

err = repo.Flush()
if err != nil {
return nil, restic.ID{}, err
}

err = repo.SaveIndex()
if err != nil {
return nil, restic.ID{}, err
}

return sn, id, nil
}
44 changes: 28 additions & 16 deletions src/restic/archive_reader_test.go → src/restic/archiver/archive_reader_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,25 @@
package restic
package archiver

import (
"bytes"
"io"
"math/rand"
"restic/backend"
"restic/pack"
"restic"
"restic/repository"
"testing"

"github.com/restic/chunker"
)

func loadBlob(t *testing.T, repo *repository.Repository, id backend.ID, buf []byte) []byte {
buf, err := repo.LoadBlob(pack.Data, id, buf)
func loadBlob(t *testing.T, repo restic.Repository, id restic.ID, buf []byte) int {
n, err := repo.LoadBlob(restic.DataBlob, id, buf)
if err != nil {
t.Fatalf("LoadBlob(%v) returned error %v", id, err)
}

return buf
return n
}

func checkSavedFile(t *testing.T, repo *repository.Repository, treeID backend.ID, name string, rd io.Reader) {
tree, err := LoadTree(repo, treeID)
func checkSavedFile(t *testing.T, repo restic.Repository, treeID restic.ID, name string, rd io.Reader) {
tree, err := repo.LoadTree(treeID)
if err != nil {
t.Fatalf("LoadTree() returned error %v", err)
}
Expand All @@ -41,20 +38,35 @@ func checkSavedFile(t *testing.T, repo *repository.Repository, treeID backend.ID
}

// check blobs
buf := make([]byte, chunker.MaxSize)
buf2 := make([]byte, chunker.MaxSize)
for i, id := range node.Content {
buf = loadBlob(t, repo, id, buf)
size, err := repo.LookupBlobSize(id, restic.DataBlob)
if err != nil {
t.Fatal(err)
}

buf := make([]byte, int(size))
n := loadBlob(t, repo, id, buf)
if n != len(buf) {
t.Errorf("wrong number of bytes read, want %d, got %d", len(buf), n)
}

buf2 = buf2[:len(buf)]
buf2 := make([]byte, int(size))
_, err = io.ReadFull(rd, buf2)
if err != nil {
t.Fatal(err)
}

if !bytes.Equal(buf, buf2) {
t.Fatalf("blob %d (%v) is wrong", i, id.Str())
}
}
}

// fakeFile returns a reader which yields deterministic pseudo-random data.
func fakeFile(t testing.TB, seed, size int64) io.Reader {
return io.LimitReader(restic.NewRandReader(rand.New(rand.NewSource(seed))), size)
}

func TestArchiveReader(t *testing.T) {
repo, cleanup := repository.TestRepository(t)
defer cleanup()
Expand All @@ -65,7 +77,7 @@ func TestArchiveReader(t *testing.T) {

f := fakeFile(t, seed, size)

sn, id, err := ArchiveReader(repo, nil, f, "fakefile")
sn, id, err := ArchiveReader(repo, nil, f, "fakefile", []string{"test"})
if err != nil {
t.Fatalf("ArchiveReader() returned error %v", err)
}
Expand Down Expand Up @@ -95,7 +107,7 @@ func BenchmarkArchiveReader(t *testing.B) {
t.ResetTimer()

for i := 0; i < t.N; i++ {
_, _, err := ArchiveReader(repo, nil, bytes.NewReader(buf), "fakefile")
_, _, err := ArchiveReader(repo, nil, bytes.NewReader(buf), "fakefile", []string{"test"})
if err != nil {
t.Fatal(err)
}
Expand Down
284 changes: 140 additions & 144 deletions src/restic/archiver.go → src/restic/archiver/archiver.go
View file

Large diffs are not rendered by default.

39 changes: 20 additions & 19 deletions src/restic/archiver_duplication_test.go → ...tic/archiver/archiver_duplication_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,32 +1,33 @@
package restic_test
package archiver_test

import (
"crypto/rand"
"errors"
"io"
mrand "math/rand"
"sync"
"testing"
"time"

"restic/errors"

"restic"
"restic/backend"
"restic/pack"
"restic/archiver"
"restic/mock"
"restic/repository"
)

const parallelSaves = 50
const testSaveIndexTime = 100 * time.Millisecond
const testTimeout = 2 * time.Second

var DupID backend.ID
var DupID restic.ID

func randomID() backend.ID {
func randomID() restic.ID {
if mrand.Float32() < 0.5 {
return DupID
}

id := backend.ID{}
id := restic.ID{}
_, err := io.ReadFull(rand.Reader, id[:])
if err != nil {
panic(err)
Expand All @@ -35,30 +36,30 @@ func randomID() backend.ID {
}

// forgetfulBackend returns a backend that forgets everything.
func forgetfulBackend() backend.Backend {
be := &backend.MockBackend{}
func forgetfulBackend() restic.Backend {
be := &mock.Backend{}

be.TestFn = func(t backend.Type, name string) (bool, error) {
be.TestFn = func(t restic.FileType, name string) (bool, error) {
return false, nil
}

be.LoadFn = func(h backend.Handle, p []byte, off int64) (int, error) {
be.LoadFn = func(h restic.Handle, p []byte, off int64) (int, error) {
return 0, errors.New("not found")
}

be.SaveFn = func(h backend.Handle, p []byte) error {
be.SaveFn = func(h restic.Handle, p []byte) error {
return nil
}

be.StatFn = func(h backend.Handle) (backend.BlobInfo, error) {
return backend.BlobInfo{}, errors.New("not found")
be.StatFn = func(h restic.Handle) (restic.FileInfo, error) {
return restic.FileInfo{}, errors.New("not found")
}

be.RemoveFn = func(t backend.Type, name string) error {
be.RemoveFn = func(t restic.FileType, name string) error {
return nil
}

be.ListFn = func(t backend.Type, done <-chan struct{}) <-chan string {
be.ListFn = func(t restic.FileType, done <-chan struct{}) <-chan string {
ch := make(chan string)
close(ch)
return ch
Expand All @@ -84,7 +85,7 @@ func testArchiverDuplication(t *testing.T) {
t.Fatal(err)
}

arch := restic.NewArchiver(repo)
arch := archiver.New(repo)

wg := &sync.WaitGroup{}
done := make(chan struct{})
Expand All @@ -101,13 +102,13 @@ func testArchiverDuplication(t *testing.T) {

id := randomID()

if repo.Index().Has(id) {
if repo.Index().Has(id, restic.DataBlob) {
continue
}

buf := make([]byte, 50)

err := arch.Save(pack.Data, buf, id)
err := arch.Save(restic.DataBlob, buf, id)
if err != nil {
t.Fatal(err)
}
Expand Down
9 changes: 5 additions & 4 deletions src/restic/archiver_int_test.go → src/restic/archiver/archiver_int_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
package restic
package archiver

import (
"os"
"testing"

"restic/pipe"
"restic/walk"
)

var treeJobs = []string{
Expand Down Expand Up @@ -82,12 +83,12 @@ func (j testPipeJob) Error() error { return j.err }
func (j testPipeJob) Info() os.FileInfo { return j.fi }
func (j testPipeJob) Result() chan<- pipe.Result { return j.res }

func testTreeWalker(done <-chan struct{}, out chan<- WalkTreeJob) {
func testTreeWalker(done <-chan struct{}, out chan<- walk.TreeJob) {
for _, e := range treeJobs {
select {
case <-done:
return
case out <- WalkTreeJob{Path: e}:
case out <- walk.TreeJob{Path: e}:
}
}

Expand All @@ -109,7 +110,7 @@ func testPipeWalker(done <-chan struct{}, out chan<- pipe.Job) {
func TestArchivePipe(t *testing.T) {
done := make(chan struct{})

treeCh := make(chan WalkTreeJob)
treeCh := make(chan walk.TreeJob)
pipeCh := make(chan pipe.Job)

go testTreeWalker(done, treeCh)
Expand Down
119 changes: 43 additions & 76 deletions src/restic/archiver_test.go → src/restic/archiver/archiver_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package restic_test
package archiver_test

import (
"bytes"
Expand All @@ -7,13 +7,14 @@ import (
"time"

"restic"
"restic/backend"
"restic/archiver"
"restic/checker"
"restic/crypto"
"restic/pack"
"restic/repository"
. "restic/test"

"restic/errors"

"github.com/restic/chunker"
)

Expand All @@ -31,7 +32,7 @@ func benchmarkChunkEncrypt(b testing.TB, buf, buf2 []byte, rd Rdr, key *crypto.K
for {
chunk, err := ch.Next(buf)

if err == io.EOF {
if errors.Cause(err) == io.EOF {
break
}

Expand All @@ -47,8 +48,8 @@ func benchmarkChunkEncrypt(b testing.TB, buf, buf2 []byte, rd Rdr, key *crypto.K
}

func BenchmarkChunkEncrypt(b *testing.B) {
repo := SetupRepo()
defer TeardownRepo(repo)
repo, cleanup := repository.TestRepository(b)
defer cleanup()

data := Random(23, 10<<20) // 10MiB
rd := bytes.NewReader(data)
Expand All @@ -69,7 +70,7 @@ func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *crypto.Key)

for {
chunk, err := ch.Next(buf)
if err == io.EOF {
if errors.Cause(err) == io.EOF {
break
}

Expand All @@ -79,8 +80,8 @@ func benchmarkChunkEncryptP(b *testing.PB, buf []byte, rd Rdr, key *crypto.Key)
}

func BenchmarkChunkEncryptParallel(b *testing.B) {
repo := SetupRepo()
defer TeardownRepo(repo)
repo, cleanup := repository.TestRepository(b)
defer cleanup()

data := Random(23, 10<<20) // 10MiB

Expand All @@ -98,12 +99,12 @@ func BenchmarkChunkEncryptParallel(b *testing.B) {
}

func archiveDirectory(b testing.TB) {
repo := SetupRepo()
defer TeardownRepo(repo)
repo, cleanup := repository.TestRepository(b)
defer cleanup()

arch := restic.NewArchiver(repo)
arch := archiver.New(repo)

_, id, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil)
_, id, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil, nil)
OK(b, err)

b.Logf("snapshot archived as %v", id)
Expand All @@ -127,9 +128,17 @@ func BenchmarkArchiveDirectory(b *testing.B) {
}
}

func countPacks(repo restic.Repository, t restic.FileType) (n uint) {
for _ = range repo.Backend().List(t, nil) {
n++
}

return n
}

func archiveWithDedup(t testing.TB) {
repo := SetupRepo()
defer TeardownRepo(repo)
repo, cleanup := repository.TestRepository(t)
defer cleanup()

if BenchArchiveDirectory == "" {
t.Skip("benchdir not set, skipping TestArchiverDedup")
Expand All @@ -142,24 +151,24 @@ func archiveWithDedup(t testing.TB) {
}

// archive a few files
sn := SnapshotDir(t, repo, BenchArchiveDirectory, nil)
sn := archiver.TestSnapshot(t, repo, BenchArchiveDirectory, nil)
t.Logf("archived snapshot %v", sn.ID().Str())

// get archive stats
cnt.before.packs = repo.Count(backend.Data)
cnt.before.dataBlobs = repo.Index().Count(pack.Data)
cnt.before.treeBlobs = repo.Index().Count(pack.Tree)
cnt.before.packs = countPacks(repo, restic.DataFile)
cnt.before.dataBlobs = repo.Index().Count(restic.DataBlob)
cnt.before.treeBlobs = repo.Index().Count(restic.TreeBlob)
t.Logf("packs %v, data blobs %v, tree blobs %v",
cnt.before.packs, cnt.before.dataBlobs, cnt.before.treeBlobs)

// archive the same files again, without parent snapshot
sn2 := SnapshotDir(t, repo, BenchArchiveDirectory, nil)
sn2 := archiver.TestSnapshot(t, repo, BenchArchiveDirectory, nil)
t.Logf("archived snapshot %v", sn2.ID().Str())

// get archive stats again
cnt.after.packs = repo.Count(backend.Data)
cnt.after.dataBlobs = repo.Index().Count(pack.Data)
cnt.after.treeBlobs = repo.Index().Count(pack.Tree)
cnt.after.packs = countPacks(repo, restic.DataFile)
cnt.after.dataBlobs = repo.Index().Count(restic.DataBlob)
cnt.after.treeBlobs = repo.Index().Count(restic.TreeBlob)
t.Logf("packs %v, data blobs %v, tree blobs %v",
cnt.after.packs, cnt.after.dataBlobs, cnt.after.treeBlobs)

Expand All @@ -170,13 +179,13 @@ func archiveWithDedup(t testing.TB) {
}

// archive the same files again, with a parent snapshot
sn3 := SnapshotDir(t, repo, BenchArchiveDirectory, sn2.ID())
sn3 := archiver.TestSnapshot(t, repo, BenchArchiveDirectory, sn2.ID())
t.Logf("archived snapshot %v, parent %v", sn3.ID().Str(), sn2.ID().Str())

// get archive stats again
cnt.after2.packs = repo.Count(backend.Data)
cnt.after2.dataBlobs = repo.Index().Count(pack.Data)
cnt.after2.treeBlobs = repo.Index().Count(pack.Tree)
cnt.after2.packs = countPacks(repo, restic.DataFile)
cnt.after2.dataBlobs = repo.Index().Count(restic.DataBlob)
cnt.after2.treeBlobs = repo.Index().Count(restic.TreeBlob)
t.Logf("packs %v, data blobs %v, tree blobs %v",
cnt.after2.packs, cnt.after2.dataBlobs, cnt.after2.treeBlobs)

Expand All @@ -191,48 +200,6 @@ func TestArchiveDedup(t *testing.T) {
archiveWithDedup(t)
}

func BenchmarkLoadTree(t *testing.B) {
repo := SetupRepo()
defer TeardownRepo(repo)

if BenchArchiveDirectory == "" {
t.Skip("benchdir not set, skipping TestArchiverDedup")
}

// archive a few files
arch := restic.NewArchiver(repo)
sn, _, err := arch.Snapshot(nil, []string{BenchArchiveDirectory}, nil)
OK(t, err)
t.Logf("archived snapshot %v", sn.ID())

list := make([]backend.ID, 0, 10)
done := make(chan struct{})

for _, idx := range repo.Index().All() {
for blob := range idx.Each(done) {
if blob.Type != pack.Tree {
continue
}

list = append(list, blob.ID)
if len(list) == cap(list) {
close(done)
break
}
}
}

// start benchmark
t.ResetTimer()

for i := 0; i < t.N; i++ {
for _, id := range list {
_, err := restic.LoadTree(repo, id)
OK(t, err)
}
}
}

// Saves several identical chunks concurrently and later checks that there are no
// unreferenced packs in the repository. See also #292 and #358.
func TestParallelSaveWithDuplication(t *testing.T) {
Expand All @@ -242,13 +209,13 @@ func TestParallelSaveWithDuplication(t *testing.T) {
}

func testParallelSaveWithDuplication(t *testing.T, seed int) {
repo := SetupRepo()
defer TeardownRepo(repo)
repo, cleanup := repository.TestRepository(t)
defer cleanup()

dataSizeMb := 128
duplication := 7

arch := restic.NewArchiver(repo)
arch := archiver.New(repo)
chunks := getRandomData(seed, dataSizeMb*1024*1024)

errChannels := [](<-chan error){}
Expand All @@ -265,9 +232,9 @@ func testParallelSaveWithDuplication(t *testing.T, seed int) {
go func(c chunker.Chunk, errChan chan<- error) {
barrier <- struct{}{}

id := backend.Hash(c.Data)
id := restic.Hash(c.Data)
time.Sleep(time.Duration(id[0]))
err := arch.Save(pack.Data, c.Data, id)
err := arch.Save(restic.DataBlob, c.Data, id)
<-barrier
errChan <- err
}(c, errChan)
Expand All @@ -292,7 +259,7 @@ func getRandomData(seed int, size int) []chunker.Chunk {

for {
c, err := chunker.Next(nil)
if err == io.EOF {
if errors.Cause(err) == io.EOF {
break
}
chunks = append(chunks, c)
Expand All @@ -301,7 +268,7 @@ func getRandomData(seed int, size int) []chunker.Chunk {
return chunks
}

func createAndInitChecker(t *testing.T, repo *repository.Repository) *checker.Checker {
func createAndInitChecker(t *testing.T, repo restic.Repository) *checker.Checker {
chkr := checker.New(repo)

hints, errs := chkr.LoadIndex()
Expand Down
21 changes: 21 additions & 0 deletions src/restic/archiver/buffer_pool.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
package archiver

import (
"sync"

"github.com/restic/chunker"
)

var bufPool = sync.Pool{
New: func() interface{} {
return make([]byte, chunker.MinSize)
},
}

func getBuf() []byte {
return bufPool.Get().([]byte)
}

func freeBuf(data []byte) {
bufPool.Put(data)
}
16 changes: 16 additions & 0 deletions src/restic/archiver/testing.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
package archiver

import (
"restic"
"testing"
)

// TestSnapshot creates a new snapshot of path.
func TestSnapshot(t testing.TB, repo restic.Repository, path string, parent *restic.ID) *restic.Snapshot {
arch := New(repo)
sn, _, err := arch.Snapshot(nil, []string{path}, []string{"test"}, parent)
if err != nil {
t.Fatal(err)
}
return sn
}
38 changes: 38 additions & 0 deletions src/restic/backend.go
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
package restic

// Backend is used to store and access data.
type Backend interface {
// Location returns a string that describes the type and location of the
// repository.
Location() string

// Test a boolean value whether a File with the name and type exists.
Test(t FileType, name string) (bool, error)

// Remove removes a File with type t and name.
Remove(t FileType, name string) error

// Close the backend
Close() error

// Load returns the data stored in the backend for h at the given offset
// and saves it in p. Load has the same semantics as io.ReaderAt, except
// that a negative offset is also allowed. In this case it references a
// position relative to the end of the file (similar to Seek()).
Load(h Handle, p []byte, off int64) (int, error)

// Save stores the data in the backend under the given handle.
Save(h Handle, p []byte) error

// Stat returns information about the File identified by h.
Stat(h Handle) (FileInfo, error)

// List returns a channel that yields all names of files of type t in an
// arbitrary order. A goroutine is started for this. If the channel done is
// closed, sending stops.
List(t FileType, done <-chan struct{}) <-chan string
}

// FileInfo is returned by Stat() and contains information about a file in the
// backend.
type FileInfo struct{ Size int64 }
5 changes: 2 additions & 3 deletions src/restic/backend/doc.go
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,4 @@
// Package backend provides local and remote storage for restic repositories.
// All backends need to implement the Backend interface. There is a
// MockBackend, which can be used for mocking in tests, and a MemBackend, which
// stores all data in a hash internally.
// All backends need to implement the Backend interface. There is a MemBackend,
// which stores all data in a map internally and can be used for testing.
package backend
61 changes: 0 additions & 61 deletions src/restic/backend/generic_test.go
View file

This file was deleted.

48 changes: 0 additions & 48 deletions src/restic/backend/handle.go
View file

This file was deleted.

58 changes: 0 additions & 58 deletions src/restic/backend/ids_test.go
View file

This file was deleted.

35 changes: 0 additions & 35 deletions src/restic/backend/idset_test.go
View file

This file was deleted.

61 changes: 0 additions & 61 deletions src/restic/backend/interface.go
View file

This file was deleted.

7 changes: 7 additions & 0 deletions src/restic/backend/local/backend_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,13 @@ func TestLocalBackendLoad(t *testing.T) {
test.TestLoad(t)
}

func TestLocalBackendLoadNegativeOffset(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestLoadNegativeOffset(t)
}

func TestLocalBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
Expand Down
3 changes: 2 additions & 1 deletion src/restic/backend/local/config.go
View file
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
package local

import (
"errors"
"strings"

"restic/errors"
)

// ParseConfig parses a local backend config.
Expand Down
142 changes: 81 additions & 61 deletions src/restic/backend/local/local.go
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
package local

import (
"errors"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"restic"

"restic/errors"

"restic/backend"
"restic/debug"
Expand All @@ -18,6 +19,8 @@ type Local struct {
p string
}

var _ restic.Backend = &Local{}

func paths(dir string) []string {
return []string{
dir,
Expand All @@ -34,8 +37,8 @@ func paths(dir string) []string {
func Open(dir string) (*Local, error) {
// test if all necessary dirs are there
for _, d := range paths(dir) {
if _, err := os.Stat(d); err != nil {
return nil, fmt.Errorf("%s does not exist", d)
if _, err := fs.Stat(d); err != nil {
return nil, errors.Wrap(err, "Open")
}
}

Expand All @@ -46,16 +49,16 @@ func Open(dir string) (*Local, error) {
// backend at dir. Afterwards a new config blob should be created.
func Create(dir string) (*Local, error) {
// test if config file already exists
_, err := os.Lstat(filepath.Join(dir, backend.Paths.Config))
_, err := fs.Lstat(filepath.Join(dir, backend.Paths.Config))
if err == nil {
return nil, errors.New("config file already exists")
}

// create paths for data, refs and temp
for _, d := range paths(dir) {
err := os.MkdirAll(d, backend.Modes.Dir)
err := fs.MkdirAll(d, backend.Modes.Dir)
if err != nil {
return nil, err
return nil, errors.Wrap(err, "MkdirAll")
}
}

Expand All @@ -69,59 +72,66 @@ func (b *Local) Location() string {
}

// Construct path for given Type and name.
func filename(base string, t backend.Type, name string) string {
if t == backend.Config {
func filename(base string, t restic.FileType, name string) string {
if t == restic.ConfigFile {
return filepath.Join(base, "config")
}

return filepath.Join(dirname(base, t, name), name)
}

// Construct directory for given Type.
func dirname(base string, t backend.Type, name string) string {
func dirname(base string, t restic.FileType, name string) string {
var n string
switch t {
case backend.Data:
case restic.DataFile:
n = backend.Paths.Data
if len(name) > 2 {
n = filepath.Join(n, name[:2])
}
case backend.Snapshot:
case restic.SnapshotFile:
n = backend.Paths.Snapshots
case backend.Index:
case restic.IndexFile:
n = backend.Paths.Index
case backend.Lock:
case restic.LockFile:
n = backend.Paths.Locks
case backend.Key:
case restic.KeyFile:
n = backend.Paths.Keys
}
return filepath.Join(base, n)
}

// Load returns the data stored in the backend for h at the given offset
// and saves it in p. Load has the same semantics as io.ReaderAt.
func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) {
// Load returns the data stored in the backend for h at the given offset and
// saves it in p. Load has the same semantics as io.ReaderAt, with one
// exception: when off is lower than zero, it is treated as an offset relative
// to the end of the file.
func (b *Local) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
debug.Log("Load %v, length %v at %v", h, len(p), off)
if err := h.Valid(); err != nil {
return 0, err
}

f, err := os.Open(filename(b.p, h.Type, h.Name))
f, err := fs.Open(filename(b.p, h.Type, h.Name))
if err != nil {
return 0, err
return 0, errors.Wrap(err, "Open")
}

defer func() {
e := f.Close()
if err == nil && e != nil {
err = e
err = errors.Wrap(e, "Close")
}
}()

if off > 0 {
switch {
case off > 0:
_, err = f.Seek(off, 0)
if err != nil {
return 0, err
}
case off < 0:
_, err = f.Seek(off, 2)
}

if err != nil {
return 0, errors.Wrap(err, "Seek")
}

return io.ReadFull(f, p)
Expand All @@ -131,130 +141,137 @@ func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) {
func writeToTempfile(tempdir string, p []byte) (filename string, err error) {
tmpfile, err := ioutil.TempFile(tempdir, "temp-")
if err != nil {
return "", err
return "", errors.Wrap(err, "TempFile")
}

n, err := tmpfile.Write(p)
if err != nil {
return "", err
return "", errors.Wrap(err, "Write")
}

if n != len(p) {
return "", errors.New("not all bytes writen")
}

if err = tmpfile.Sync(); err != nil {
return "", err
return "", errors.Wrap(err, "Syncn")
}

err = fs.ClearCache(tmpfile)
if err != nil {
return "", err
return "", errors.Wrap(err, "ClearCache")
}

err = tmpfile.Close()
if err != nil {
return "", err
return "", errors.Wrap(err, "Close")
}

return tmpfile.Name(), nil
}

// Save stores data in the backend at the handle.
func (b *Local) Save(h backend.Handle, p []byte) (err error) {
func (b *Local) Save(h restic.Handle, p []byte) (err error) {
debug.Log("Save %v, length %v", h, len(p))
if err := h.Valid(); err != nil {
return err
}

tmpfile, err := writeToTempfile(filepath.Join(b.p, backend.Paths.Temp), p)
debug.Log("local.Save", "saved %v (%d bytes) to %v", h, len(p), tmpfile)
debug.Log("saved %v (%d bytes) to %v", h, len(p), tmpfile)
if err != nil {
return err
}

filename := filename(b.p, h.Type, h.Name)

// test if new path already exists
if _, err := os.Stat(filename); err == nil {
return fmt.Errorf("Rename(): file %v already exists", filename)
if _, err := fs.Stat(filename); err == nil {
return errors.Errorf("Rename(): file %v already exists", filename)
}

// create directories if necessary, ignore errors
if h.Type == backend.Data {
err = os.MkdirAll(filepath.Dir(filename), backend.Modes.Dir)
if h.Type == restic.DataFile {
err = fs.MkdirAll(filepath.Dir(filename), backend.Modes.Dir)
if err != nil {
return err
return errors.Wrap(err, "MkdirAll")
}
}

err = os.Rename(tmpfile, filename)
debug.Log("local.Save", "save %v: rename %v -> %v: %v",
err = fs.Rename(tmpfile, filename)
debug.Log("save %v: rename %v -> %v: %v",
h, filepath.Base(tmpfile), filepath.Base(filename), err)

if err != nil {
return err
return errors.Wrap(err, "Rename")
}

// set mode to read-only
fi, err := os.Stat(filename)
fi, err := fs.Stat(filename)
if err != nil {
return err
return errors.Wrap(err, "Stat")
}

return setNewFileMode(filename, fi)
}

// Stat returns information about a blob.
func (b *Local) Stat(h backend.Handle) (backend.BlobInfo, error) {
func (b *Local) Stat(h restic.Handle) (restic.FileInfo, error) {
debug.Log("Stat %v", h)
if err := h.Valid(); err != nil {
return backend.BlobInfo{}, err
return restic.FileInfo{}, err
}

fi, err := os.Stat(filename(b.p, h.Type, h.Name))
fi, err := fs.Stat(filename(b.p, h.Type, h.Name))
if err != nil {
return backend.BlobInfo{}, err
return restic.FileInfo{}, errors.Wrap(err, "Stat")
}

return backend.BlobInfo{Size: fi.Size()}, nil
return restic.FileInfo{Size: fi.Size()}, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (b *Local) Test(t backend.Type, name string) (bool, error) {
_, err := os.Stat(filename(b.p, t, name))
func (b *Local) Test(t restic.FileType, name string) (bool, error) {
debug.Log("Test %v %v", t, name)
_, err := fs.Stat(filename(b.p, t, name))
if err != nil {
if os.IsNotExist(err) {
if os.IsNotExist(errors.Cause(err)) {
return false, nil
}
return false, err
return false, errors.Wrap(err, "Stat")
}

return true, nil
}

// Remove removes the blob with the given name and type.
func (b *Local) Remove(t backend.Type, name string) error {
func (b *Local) Remove(t restic.FileType, name string) error {
debug.Log("Remove %v %v", t, name)
fn := filename(b.p, t, name)

// reset read-only flag
err := os.Chmod(fn, 0666)
err := fs.Chmod(fn, 0666)
if err != nil {
return err
return errors.Wrap(err, "Chmod")
}

return os.Remove(fn)
return fs.Remove(fn)
}

func isFile(fi os.FileInfo) bool {
return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0
}

func readdir(d string) (fileInfos []os.FileInfo, err error) {
f, e := os.Open(d)
f, e := fs.Open(d)
if e != nil {
return nil, e
return nil, errors.Wrap(e, "Open")
}

defer func() {
e := f.Close()
if err == nil {
err = e
err = errors.Wrap(e, "Close")
}
}()

Expand Down Expand Up @@ -303,9 +320,10 @@ func listDirs(dir string) (filenames []string, err error) {
// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string {
func (b *Local) List(t restic.FileType, done <-chan struct{}) <-chan string {
debug.Log("List %v", t)
lister := listDir
if t == backend.Data {
if t == restic.DataFile {
lister = listDirs
}

Expand Down Expand Up @@ -336,11 +354,13 @@ func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string {

// Delete removes the repository and all files.
func (b *Local) Delete() error {
return os.RemoveAll(b.p)
debug.Log("Delete()")
return fs.RemoveAll(b.p)
}

// Close closes all open files.
func (b *Local) Close() error {
debug.Log("Close()")
// this does not need to do anything, all open files are closed within the
// same function.
return nil
Expand Down
6 changes: 3 additions & 3 deletions src/restic/backend/local/local_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import (
"fmt"
"io/ioutil"
"os"
"restic"

"restic/backend"
"restic/backend/local"
"restic/backend/test"
)
Expand All @@ -30,15 +30,15 @@ func createTempdir() error {
}

func init() {
test.CreateFn = func() (backend.Backend, error) {
test.CreateFn = func() (restic.Backend, error) {
err := createTempdir()
if err != nil {
return nil, err
}
return local.Create(tempBackendDir)
}

test.OpenFn = func() (backend.Backend, error) {
test.OpenFn = func() (restic.Backend, error) {
err := createTempdir()
if err != nil {
return nil, err
Expand Down
3 changes: 2 additions & 1 deletion src/restic/backend/local/local_unix.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,10 @@ package local

import (
"os"
"restic/fs"
)

// set file to readonly
func setNewFileMode(f string, fi os.FileInfo) error {
return os.Chmod(f, fi.Mode()&os.FileMode(^uint32(0222)))
return fs.Chmod(f, fi.Mode()&os.FileMode(^uint32(0222)))
}
7 changes: 7 additions & 0 deletions src/restic/backend/mem/backend_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,13 @@ func TestMemBackendLoad(t *testing.T) {
test.TestLoad(t)
}

func TestMemBackendLoadNegativeOffset(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestLoadNegativeOffset(t)
}

func TestMemBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
Expand Down
130 changes: 57 additions & 73 deletions src/restic/backend/mem/mem_backend.go
View file
Original file line number Diff line number Diff line change
@@ -1,28 +1,30 @@
package mem

import (
"errors"
"io"
"restic"
"sync"

"restic/backend"
"restic/errors"

"restic/debug"
)

type entry struct {
Type backend.Type
Type restic.FileType
Name string
}

type memMap map[entry][]byte

// make sure that MemoryBackend implements backend.Backend
var _ restic.Backend = &MemoryBackend{}

// MemoryBackend is a mock backend that uses a map for storing all data in
// memory. This should only be used for tests.
type MemoryBackend struct {
data memMap
m sync.Mutex

backend.MockBackend
}

// New returns a new backend that saves all data in a map in memory.
Expand All @@ -31,64 +33,17 @@ func New() *MemoryBackend {
data: make(memMap),
}

be.MockBackend.TestFn = func(t backend.Type, name string) (bool, error) {
return memTest(be, t, name)
}

be.MockBackend.LoadFn = func(h backend.Handle, p []byte, off int64) (int, error) {
return memLoad(be, h, p, off)
}

be.MockBackend.SaveFn = func(h backend.Handle, p []byte) error {
return memSave(be, h, p)
}

be.MockBackend.StatFn = func(h backend.Handle) (backend.BlobInfo, error) {
return memStat(be, h)
}

be.MockBackend.RemoveFn = func(t backend.Type, name string) error {
return memRemove(be, t, name)
}

be.MockBackend.ListFn = func(t backend.Type, done <-chan struct{}) <-chan string {
return memList(be, t, done)
}

be.MockBackend.DeleteFn = func() error {
be.m.Lock()
defer be.m.Unlock()

be.data = make(memMap)
return nil
}

be.MockBackend.LocationFn = func() string {
return "Memory Backend"
}

debug.Log("MemoryBackend.New", "created new memory backend")
debug.Log("created new memory backend")

return be
}

func (be *MemoryBackend) insert(t backend.Type, name string, data []byte) error {
// Test returns whether a file exists.
func (be *MemoryBackend) Test(t restic.FileType, name string) (bool, error) {
be.m.Lock()
defer be.m.Unlock()

if _, ok := be.data[entry{t, name}]; ok {
return errors.New("already present")
}

be.data[entry{t, name}] = data
return nil
}

func memTest(be *MemoryBackend, t backend.Type, name string) (bool, error) {
be.m.Lock()
defer be.m.Unlock()

debug.Log("MemoryBackend.Test", "test %v %v", t, name)
debug.Log("test %v %v", t, name)

if _, ok := be.data[entry{t, name}]; ok {
return true, nil
Expand All @@ -97,27 +52,33 @@ func memTest(be *MemoryBackend, t backend.Type, name string) (bool, error) {
return false, nil
}

func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, error) {
// Load reads data from the backend.
func (be *MemoryBackend) Load(h restic.Handle, p []byte, off int64) (int, error) {
if err := h.Valid(); err != nil {
return 0, err
}

be.m.Lock()
defer be.m.Unlock()

if h.Type == backend.Config {
if h.Type == restic.ConfigFile {
h.Name = ""
}

debug.Log("MemoryBackend.Load", "get %v offset %v len %v", h, off, len(p))
debug.Log("get %v offset %v len %v", h, off, len(p))

if _, ok := be.data[entry{h.Type, h.Name}]; !ok {
return 0, errors.New("no such data")
}

buf := be.data[entry{h.Type, h.Name}]
if off > int64(len(buf)) {
switch {
case off > int64(len(buf)):
return 0, errors.New("offset beyond end of file")
case off < -int64(len(buf)):
off = 0
case off < 0:
off = int64(len(buf)) + off
}

buf = buf[off:]
Expand All @@ -131,57 +92,60 @@ func memLoad(be *MemoryBackend, h backend.Handle, p []byte, off int64) (int, err
return n, nil
}

func memSave(be *MemoryBackend, h backend.Handle, p []byte) error {
// Save adds new Data to the backend.
func (be *MemoryBackend) Save(h restic.Handle, p []byte) error {
if err := h.Valid(); err != nil {
return err
}

be.m.Lock()
defer be.m.Unlock()

if h.Type == backend.Config {
if h.Type == restic.ConfigFile {
h.Name = ""
}

if _, ok := be.data[entry{h.Type, h.Name}]; ok {
return errors.New("file already exists")
}

debug.Log("MemoryBackend.Save", "save %v bytes at %v", len(p), h)
debug.Log("save %v bytes at %v", len(p), h)
buf := make([]byte, len(p))
copy(buf, p)
be.data[entry{h.Type, h.Name}] = buf

return nil
}

func memStat(be *MemoryBackend, h backend.Handle) (backend.BlobInfo, error) {
// Stat returns information about a file in the backend.
func (be *MemoryBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
be.m.Lock()
defer be.m.Unlock()

if err := h.Valid(); err != nil {
return backend.BlobInfo{}, err
return restic.FileInfo{}, err
}

if h.Type == backend.Config {
if h.Type == restic.ConfigFile {
h.Name = ""
}

debug.Log("MemoryBackend.Stat", "stat %v", h)
debug.Log("stat %v", h)

e, ok := be.data[entry{h.Type, h.Name}]
if !ok {
return backend.BlobInfo{}, errors.New("no such data")
return restic.FileInfo{}, errors.New("no such data")
}

return backend.BlobInfo{Size: int64(len(e))}, nil
return restic.FileInfo{Size: int64(len(e))}, nil
}

func memRemove(be *MemoryBackend, t backend.Type, name string) error {
// Remove deletes a file from the backend.
func (be *MemoryBackend) Remove(t restic.FileType, name string) error {
be.m.Lock()
defer be.m.Unlock()

debug.Log("MemoryBackend.Remove", "get %v %v", t, name)
debug.Log("get %v %v", t, name)

if _, ok := be.data[entry{t, name}]; !ok {
return errors.New("no such data")
Expand All @@ -192,7 +156,8 @@ func memRemove(be *MemoryBackend, t backend.Type, name string) error {
return nil
}

func memList(be *MemoryBackend, t backend.Type, done <-chan struct{}) <-chan string {
// List returns a channel which yields entries from the backend.
func (be *MemoryBackend) List(t restic.FileType, done <-chan struct{}) <-chan string {
be.m.Lock()
defer be.m.Unlock()

Expand All @@ -206,7 +171,7 @@ func memList(be *MemoryBackend, t backend.Type, done <-chan struct{}) <-chan str
ids = append(ids, entry.Name)
}

debug.Log("MemoryBackend.List", "list %v: %v", t, ids)
debug.Log("list %v: %v", t, ids)

go func() {
defer close(ch)
Expand All @@ -221,3 +186,22 @@ func memList(be *MemoryBackend, t backend.Type, done <-chan struct{}) <-chan str

return ch
}

// Location returns the location of the backend (RAM).
func (be *MemoryBackend) Location() string {
return "RAM"
}

// Delete removes all data in the backend.
func (be *MemoryBackend) Delete() error {
be.m.Lock()
defer be.m.Unlock()

be.data = make(memMap)
return nil
}

// Close closes the backend.
func (be *MemoryBackend) Close() error {
return nil
}
11 changes: 6 additions & 5 deletions src/restic/backend/mem/mem_backend_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,19 +1,20 @@
package mem_test

import (
"errors"
"restic"

"restic/errors"

"restic/backend"
"restic/backend/mem"
"restic/backend/test"
)

var be backend.Backend
var be restic.Backend

//go:generate go run ../test/generate_backend_tests.go

func init() {
test.CreateFn = func() (backend.Backend, error) {
test.CreateFn = func() (restic.Backend, error) {
if be != nil {
return nil, errors.New("temporary memory backend dir already exists")
}
Expand All @@ -23,7 +24,7 @@ func init() {
return be, nil
}

test.OpenFn = func() (backend.Backend, error) {
test.OpenFn = func() (restic.Backend, error) {
if be == nil {
return nil, errors.New("repository not initialized")
}
Expand Down
103 changes: 0 additions & 103 deletions src/restic/backend/mock_backend.go
View file

This file was deleted.

63 changes: 0 additions & 63 deletions src/restic/backend/readseeker.go
View file

This file was deleted.

114 changes: 0 additions & 114 deletions src/restic/backend/readseeker_test.go
View file

This file was deleted.

7 changes: 7 additions & 0 deletions src/restic/backend/rest/backend_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,13 @@ func TestRestBackendLoad(t *testing.T) {
test.TestLoad(t)
}

func TestRestBackendLoadNegativeOffset(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestLoadNegativeOffset(t)
}

func TestRestBackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
Expand Down
5 changes: 3 additions & 2 deletions src/restic/backend/rest/config.go
View file
Original file line number Diff line number Diff line change
@@ -1,9 +1,10 @@
package rest

import (
"errors"
"net/url"
"strings"

"restic/errors"
)

// Config contains all configuration necessary to connect to a REST server.
Expand All @@ -21,7 +22,7 @@ func ParseConfig(s string) (interface{}, error) {
u, err := url.Parse(s)

if err != nil {
return nil, err
return nil, errors.Wrap(err, "url.Parse")
}

cfg := Config{URL: u}
Expand Down
82 changes: 49 additions & 33 deletions src/restic/backend/rest/rest.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,38 +3,40 @@ package rest
import (
"bytes"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"path"
"restic"
"strings"

"restic/errors"

"restic/backend"
)

const connLimit = 10

// restPath returns the path to the given resource.
func restPath(url *url.URL, h backend.Handle) string {
func restPath(url *url.URL, h restic.Handle) string {
u := *url

var dir string

switch h.Type {
case backend.Config:
case restic.ConfigFile:
dir = ""
h.Name = "config"
case backend.Data:
case restic.DataFile:
dir = backend.Paths.Data
case backend.Snapshot:
case restic.SnapshotFile:
dir = backend.Paths.Snapshots
case backend.Index:
case restic.IndexFile:
dir = backend.Paths.Index
case backend.Lock:
case restic.LockFile:
dir = backend.Paths.Locks
case backend.Key:
case restic.KeyFile:
dir = backend.Paths.Keys
default:
dir = string(h.Type)
Expand All @@ -52,7 +54,7 @@ type restBackend struct {
}

// Open opens the REST backend with the given config.
func Open(cfg Config) (backend.Backend, error) {
func Open(cfg Config) (restic.Backend, error) {
connChan := make(chan struct{}, connLimit)
for i := 0; i < connLimit; i++ {
connChan <- struct{}{}
Expand All @@ -70,14 +72,28 @@ func (b *restBackend) Location() string {

// Load returns the data stored in the backend for h at the given offset
// and saves it in p. Load has the same semantics as io.ReaderAt.
func (b *restBackend) Load(h backend.Handle, p []byte, off int64) (n int, err error) {
func (b *restBackend) Load(h restic.Handle, p []byte, off int64) (n int, err error) {
if err := h.Valid(); err != nil {
return 0, err
}

// invert offset
if off < 0 {
info, err := b.Stat(h)
if err != nil {
return 0, errors.Wrap(err, "Stat")
}

if -off > info.Size {
off = 0
} else {
off = info.Size + off
}
}

req, err := http.NewRequest("GET", restPath(b.url, h), nil)
if err != nil {
return 0, err
return 0, errors.Wrap(err, "http.NewRequest")
}
req.Header.Add("Range", fmt.Sprintf("bytes=%d-%d", off, off+int64(len(p))))
<-b.connChan
Expand All @@ -89,23 +105,23 @@ func (b *restBackend) Load(h backend.Handle, p []byte, off int64) (n int, err er
e := resp.Body.Close()

if err == nil {
err = e
err = errors.Wrap(e, "Close")
}
}()
}

if err != nil {
return 0, err
return 0, errors.Wrap(err, "client.Do")
}
if resp.StatusCode != 200 && resp.StatusCode != 206 {
return 0, fmt.Errorf("unexpected HTTP response code %v", resp.StatusCode)
return 0, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
}

return io.ReadFull(resp.Body, p)
}

// Save stores data in the backend at the handle.
func (b *restBackend) Save(h backend.Handle, p []byte) (err error) {
func (b *restBackend) Save(h restic.Handle, p []byte) (err error) {
if err := h.Valid(); err != nil {
return err
}
Expand All @@ -119,57 +135,57 @@ func (b *restBackend) Save(h backend.Handle, p []byte) (err error) {
e := resp.Body.Close()

if err == nil {
err = e
err = errors.Wrap(e, "Close")
}
}()
}

if err != nil {
return err
return errors.Wrap(err, "client.Post")
}

if resp.StatusCode != 200 {
return fmt.Errorf("unexpected HTTP response code %v", resp.StatusCode)
return errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
}

return nil
}

// Stat returns information about a blob.
func (b *restBackend) Stat(h backend.Handle) (backend.BlobInfo, error) {
func (b *restBackend) Stat(h restic.Handle) (restic.FileInfo, error) {
if err := h.Valid(); err != nil {
return backend.BlobInfo{}, err
return restic.FileInfo{}, err
}

<-b.connChan
resp, err := b.client.Head(restPath(b.url, h))
b.connChan <- struct{}{}
if err != nil {
return backend.BlobInfo{}, err
return restic.FileInfo{}, errors.Wrap(err, "client.Head")
}

if err = resp.Body.Close(); err != nil {
return backend.BlobInfo{}, err
return restic.FileInfo{}, errors.Wrap(err, "Close")
}

if resp.StatusCode != 200 {
return backend.BlobInfo{}, fmt.Errorf("unexpected HTTP response code %v", resp.StatusCode)
return restic.FileInfo{}, errors.Errorf("unexpected HTTP response code %v", resp.StatusCode)
}

if resp.ContentLength < 0 {
return backend.BlobInfo{}, errors.New("negative content length")
return restic.FileInfo{}, errors.New("negative content length")
}

bi := backend.BlobInfo{
bi := restic.FileInfo{
Size: resp.ContentLength,
}

return bi, nil
}

// Test returns true if a blob of the given type and name exists in the backend.
func (b *restBackend) Test(t backend.Type, name string) (bool, error) {
_, err := b.Stat(backend.Handle{Type: t, Name: name})
func (b *restBackend) Test(t restic.FileType, name string) (bool, error) {
_, err := b.Stat(restic.Handle{Type: t, Name: name})
if err != nil {
return false, nil
}
Expand All @@ -178,22 +194,22 @@ func (b *restBackend) Test(t backend.Type, name string) (bool, error) {
}

// Remove removes the blob with the given name and type.
func (b *restBackend) Remove(t backend.Type, name string) error {
h := backend.Handle{Type: t, Name: name}
func (b *restBackend) Remove(t restic.FileType, name string) error {
h := restic.Handle{Type: t, Name: name}
if err := h.Valid(); err != nil {
return err
}

req, err := http.NewRequest("DELETE", restPath(b.url, h), nil)
if err != nil {
return err
return errors.Wrap(err, "http.NewRequest")
}
<-b.connChan
resp, err := b.client.Do(req)
b.connChan <- struct{}{}

if err != nil {
return err
return errors.Wrap(err, "client.Do")
}

if resp.StatusCode != 200 {
Expand All @@ -206,10 +222,10 @@ func (b *restBackend) Remove(t backend.Type, name string) error {
// List returns a channel that yields all names of blobs of type t. A
// goroutine is started for this. If the channel done is closed, sending
// stops.
func (b *restBackend) List(t backend.Type, done <-chan struct{}) <-chan string {
func (b *restBackend) List(t restic.FileType, done <-chan struct{}) <-chan string {
ch := make(chan string)

url := restPath(b.url, backend.Handle{Type: t})
url := restPath(b.url, restic.Handle{Type: t})
if !strings.HasSuffix(url, "/") {
url += "/"
}
Expand Down
16 changes: 8 additions & 8 deletions src/restic/backend/rest/rest_path_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,35 +2,35 @@ package rest

import (
"net/url"
"restic/backend"
"restic"
"testing"
)

var restPathTests = []struct {
Handle backend.Handle
Handle restic.Handle
URL *url.URL
Result string
}{
{
URL: parseURL("https://hostname.foo"),
Handle: backend.Handle{
Type: backend.Data,
Handle: restic.Handle{
Type: restic.DataFile,
Name: "foobar",
},
Result: "https://hostname.foo/data/foobar",
},
{
URL: parseURL("https://hostname.foo:1234/prefix/repo"),
Handle: backend.Handle{
Type: backend.Lock,
Handle: restic.Handle{
Type: restic.LockFile,
Name: "foobar",
},
Result: "https://hostname.foo:1234/prefix/repo/locks/foobar",
},
{
URL: parseURL("https://hostname.foo:1234/prefix/repo"),
Handle: backend.Handle{
Type: backend.Config,
Handle: restic.Handle{
Type: restic.ConfigFile,
Name: "foobar",
},
Result: "https://hostname.foo:1234/prefix/repo/config",
Expand Down
11 changes: 6 additions & 5 deletions src/restic/backend/rest/rest_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
package rest_test

import (
"errors"
"fmt"
"net/url"
"os"
"restic"

"restic/errors"

"restic/backend"
"restic/backend/rest"
"restic/backend/test"
. "restic/test"
Expand All @@ -30,13 +31,13 @@ func init() {
URL: url,
}

test.CreateFn = func() (backend.Backend, error) {
test.CreateFn = func() (restic.Backend, error) {
be, err := rest.Open(cfg)
if err != nil {
return nil, err
}

exists, err := be.Test(backend.Config, "")
exists, err := be.Test(restic.ConfigFile, "")
if err != nil {
return nil, err
}
Expand All @@ -48,7 +49,7 @@ func init() {
return be, nil
}

test.OpenFn = func() (backend.Backend, error) {
test.OpenFn = func() (restic.Backend, error) {
return rest.Open(cfg)
}
}
7 changes: 7 additions & 0 deletions src/restic/backend/s3/backend_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,13 @@ func TestS3BackendLoad(t *testing.T) {
test.TestLoad(t)
}

func TestS3BackendLoadNegativeOffset(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
}
test.TestLoadNegativeOffset(t)
}

func TestS3BackendSave(t *testing.T) {
if SkipMessage != "" {
t.Skip(SkipMessage)
Expand Down
5 changes: 3 additions & 2 deletions src/restic/backend/s3/config.go
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,11 @@
package s3

import (
"errors"
"net/url"
"path"
"strings"

"restic/errors"
)

// Config contains all configuration necessary to connect to an s3 compatible
Expand All @@ -31,7 +32,7 @@ func ParseConfig(s string) (interface{}, error) {
// bucket name and prefix
url, err := url.Parse(s[3:])
if err != nil {
return nil, err
return nil, errors.Wrap(err, "url.Parse")
}

if url.Path == "" {
Expand Down
Loading