94 changes: 69 additions & 25 deletions src/restic/checker/checker.go
Original file line number Diff line number Diff line change
@@ -1,14 +1,17 @@
package checker

import (
"bytes"
"crypto/sha256"
"fmt"
"io"
"io/ioutil"
"os"
"sync"

"restic/errors"
"restic/hashing"

"restic"
"restic/backend"
"restic/crypto"
"restic/debug"
"restic/pack"
Expand Down Expand Up @@ -77,6 +80,7 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
debug.Log("Start")
type indexRes struct {
Index *repository.Index
err error
ID string
}

Expand All @@ -92,39 +96,40 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
idx, err = repository.LoadIndexWithDecoder(c.repo, id, repository.DecodeOldIndex)
}

if err != nil {
return err
}
err = errors.Wrapf(err, "error loading index %v", id.Str())

select {
case indexCh <- indexRes{Index: idx, ID: id.String()}:
case indexCh <- indexRes{Index: idx, ID: id.String(), err: err}:
case <-done:
}

return nil
}

var perr error
go func() {
defer close(indexCh)
debug.Log("start loading indexes in parallel")
perr = repository.FilesInParallel(c.repo.Backend(), restic.IndexFile, defaultParallelism,
err := repository.FilesInParallel(c.repo.Backend(), restic.IndexFile, defaultParallelism,
repository.ParallelWorkFuncParseID(worker))
debug.Log("loading indexes finished, error: %v", perr)
debug.Log("loading indexes finished, error: %v", err)
if err != nil {
panic(err)
}
}()

done := make(chan struct{})
defer close(done)

if perr != nil {
errs = append(errs, perr)
return hints, errs
}

packToIndex := make(map[restic.ID]restic.IDSet)

for res := range indexCh {
debug.Log("process index %v", res.ID)
debug.Log("process index %v, err %v", res.ID, res.err)

if res.err != nil {
errs = append(errs, res.err)
continue
}

idxID, err := restic.ParseID(res.ID)
if err != nil {
errs = append(errs, errors.Errorf("unable to parse as index ID: %v", res.ID))
Expand All @@ -151,8 +156,6 @@ func (c *Checker) LoadIndex() (hints []error, errs []error) {
debug.Log("%d blobs processed", cnt)
}

debug.Log("done, error %v", perr)

debug.Log("checking for duplicate packs")
for packID := range c.packs {
debug.Log(" check pack %v: contained in %d indexes", packID.Str(), len(packToIndex[packID]))
Expand Down Expand Up @@ -659,36 +662,77 @@ func (c *Checker) CountPacks() uint64 {
func checkPack(r restic.Repository, id restic.ID) error {
debug.Log("checking pack %v", id.Str())
h := restic.Handle{Type: restic.DataFile, Name: id.String()}
buf, err := backend.LoadAll(r.Backend(), h)

rd, err := r.Backend().Load(h, 0, 0)
if err != nil {
return err
}

packfile, err := ioutil.TempFile("", "restic-temp-check-")
if err != nil {
return errors.Wrap(err, "TempFile")
}

defer func() {
packfile.Close()
os.Remove(packfile.Name())
}()

hrd := hashing.NewReader(rd, sha256.New())
size, err := io.Copy(packfile, hrd)
if err != nil {
return errors.Wrap(err, "Copy")
}

if err = rd.Close(); err != nil {
return err
}

hash := restic.Hash(buf)
hash := restic.IDFromHash(hrd.Sum(nil))
debug.Log("hash for pack %v is %v", id.Str(), hash.Str())

if !hash.Equal(id) {
debug.Log("Pack ID does not match, want %v, got %v", id.Str(), hash.Str())
return errors.Errorf("Pack ID does not match, want %v, got %v", id.Str(), hash.Str())
}

blobs, err := pack.List(r.Key(), bytes.NewReader(buf), int64(len(buf)))
blobs, err := pack.List(r.Key(), packfile, size)
if err != nil {
return err
}

var errs []error
var buf []byte
for i, blob := range blobs {
debug.Log(" check blob %d: %v", i, blob.ID.Str())
debug.Log(" check blob %d: %v", i, blob)

buf = buf[:cap(buf)]
if uint(len(buf)) < blob.Length {
buf = make([]byte, blob.Length)
}
buf = buf[:blob.Length]

_, err := packfile.Seek(int64(blob.Offset), 0)
if err != nil {
return errors.Errorf("Seek(%v): %v", blob.Offset, err)
}

_, err = io.ReadFull(packfile, buf)
if err != nil {
debug.Log(" error loading blob %v: %v", blob.ID.Str(), err)
errs = append(errs, errors.Errorf("blob %v: %v", i, err))
continue
}

plainBuf := make([]byte, blob.Length)
n, err := crypto.Decrypt(r.Key(), plainBuf, buf[blob.Offset:blob.Offset+blob.Length])
n, err := crypto.Decrypt(r.Key(), buf, buf)
if err != nil {
debug.Log(" error decrypting blob %v: %v", blob.ID.Str(), err)
errs = append(errs, errors.Errorf("blob %v: %v", i, err))
continue
}
plainBuf = plainBuf[:n]
buf = buf[:n]

hash := restic.Hash(plainBuf)
hash := restic.Hash(buf)
if !hash.Equal(blob.ID) {
debug.Log(" Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str())
errs = append(errs, errors.Errorf("Blob ID does not match, want %v, got %v", blob.ID.Str(), hash.Str()))
Expand Down
69 changes: 68 additions & 1 deletion src/restic/checker/checker_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -179,6 +179,48 @@ func TestUnreferencedBlobs(t *testing.T) {
test.Equals(t, unusedBlobsBySnapshot, blobs)
}

func TestModifiedIndex(t *testing.T) {
repodir, cleanup := test.Env(t, checkerTestData)
defer cleanup()

repo := repository.TestOpenLocal(t, repodir)

done := make(chan struct{})
defer close(done)

h := restic.Handle{
Type: restic.IndexFile,
Name: "90f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
}
f, err := repo.Backend().Load(h, 0, 0)
test.OK(t, err)

// save the index again with a modified name so that the hash doesn't match
// the content any more
h2 := restic.Handle{
Type: restic.IndexFile,
Name: "80f838b4ac28735fda8644fe6a08dbc742e57aaf81b30977b4fefa357010eafd",
}
err = repo.Backend().Save(h2, f)
test.OK(t, err)

test.OK(t, f.Close())

chkr := checker.New(repo)
hints, errs := chkr.LoadIndex()
if len(errs) == 0 {
t.Fatalf("expected errors not found")
}

for _, err := range errs {
t.Logf("found expected error %v", err)
}

if len(hints) > 0 {
t.Errorf("expected no hints, got %v: %v", len(hints), hints)
}
}

var checkerDuplicateIndexTestData = filepath.Join("testdata", "duplicate-packs-in-index-test-repo.tar.gz")

func TestDuplicatePacksInIndex(t *testing.T) {
Expand Down Expand Up @@ -261,7 +303,7 @@ func TestCheckerModifiedData(t *testing.T) {
defer cleanup()

arch := archiver.New(repo)
_, id, err := arch.Snapshot(nil, []string{"."}, nil, nil)
_, id, err := arch.Snapshot(nil, []string{"."}, nil, "localhost", nil)
test.OK(t, err)
t.Logf("archived as %v", id.Str())

Expand Down Expand Up @@ -299,3 +341,28 @@ func TestCheckerModifiedData(t *testing.T) {
t.Fatal("no error found, checker is broken")
}
}

func BenchmarkChecker(t *testing.B) {
repodir, cleanup := test.Env(t, checkerTestData)
defer cleanup()

repo := repository.TestOpenLocal(t, repodir)

chkr := checker.New(repo)
hints, errs := chkr.LoadIndex()
if len(errs) > 0 {
t.Fatalf("expected no errors, got %v: %v", len(errs), errs)
}

if len(hints) > 0 {
t.Errorf("expected no hints, got %v: %v", len(hints), hints)
}

t.ResetTimer()

for i := 0; i < t.N; i++ {
test.OKs(t, checkPacks(chkr))
test.OKs(t, checkStruct(chkr))
test.OKs(t, checkData(chkr))
}
}
4 changes: 4 additions & 0 deletions src/restic/errors/wrap.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,3 +18,7 @@ var Errorf = errors.Errorf
// Wrap wraps an error retrieved from outside of restic. Wrapped so that this
// package does not appear in the stack trace.
var Wrap = errors.Wrap

// Wrapf returns an error annotating err with the format specifier. If err is
// nil, Wrapf returns nil.
var Wrapf = errors.Wrapf
2 changes: 1 addition & 1 deletion src/restic/fs/doc.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
// Package fs implements an OS independend abstraction of a file system
// Package fs implements an OS independent abstraction of a file system
// suitable for backup purposes.
package fs
6 changes: 6 additions & 0 deletions src/restic/fs/file.go
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,12 @@ func Symlink(oldname, newname string) error {
return os.Symlink(fixpath(oldname), fixpath(newname))
}

// Link creates newname as a hard link to oldname.
// If there is an error, it will be of type *LinkError.
func Link(oldname, newname string) error {
return os.Link(fixpath(oldname), fixpath(newname))
}

// Stat returns a FileInfo structure describing the named file.
// If there is an error, it will be of type *PathError.
func Stat(name string) (os.FileInfo, error) {
Expand Down
34 changes: 34 additions & 0 deletions src/restic/fuse/dir.go
Original file line number Diff line number Diff line change
Expand Up @@ -114,9 +114,25 @@ func (d *dir) Attr(ctx context.Context, a *fuse.Attr) error {
a.Atime = d.node.AccessTime
a.Ctime = d.node.ChangeTime
a.Mtime = d.node.ModTime

a.Nlink = d.calcNumberOfLinks()

return nil
}

func (d *dir) calcNumberOfLinks() uint32 {
// a directory d has 2 hardlinks + the number
// of directories contained by d
var count uint32
count = 2
for _, node := range d.items {
if node.Type == "dir" {
count++
}
}
return count
}

func (d *dir) ReadDirAll(ctx context.Context) ([]fuse.Dirent, error) {
debug.Log("called")
ret := make([]fuse.Dirent, 0, len(d.items))
Expand Down Expand Up @@ -161,3 +177,21 @@ func (d *dir) Lookup(ctx context.Context, name string) (fs.Node, error) {
return nil, fuse.ENOENT
}
}

func (d *dir) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
debug.Log("Listxattr(%v, %v)", d.node.Name, req.Size)
for _, attr := range d.node.ExtendedAttributes {
resp.Append(attr.Name)
}
return nil
}

func (d *dir) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
debug.Log("Getxattr(%v, %v, %v)", d.node.Name, req.Name, req.Size)
attrval := d.node.GetExtendedAttribute(req.Name)
if attrval != nil {
resp.Xattr = attrval
return nil
}
return fuse.ErrNoXattr
}
21 changes: 21 additions & 0 deletions src/restic/fuse/file.go
Original file line number Diff line number Diff line change
Expand Up @@ -74,6 +74,7 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
a.Size = f.node.Size
a.Blocks = (f.node.Size / blockSize) + 1
a.BlockSize = blockSize
a.Nlink = uint32(f.node.Links)

if !f.ownerIsRoot {
a.Uid = f.node.UID
Expand All @@ -82,7 +83,9 @@ func (f *file) Attr(ctx context.Context, a *fuse.Attr) error {
a.Atime = f.node.AccessTime
a.Ctime = f.node.ChangeTime
a.Mtime = f.node.ModTime

return nil

}

func (f *file) getBlobAt(i int) (blob []byte, err error) {
Expand Down Expand Up @@ -161,3 +164,21 @@ func (f *file) Release(ctx context.Context, req *fuse.ReleaseRequest) error {
}
return nil
}

func (f *file) Listxattr(ctx context.Context, req *fuse.ListxattrRequest, resp *fuse.ListxattrResponse) error {
debug.Log("Listxattr(%v, %v)", f.node.Name, req.Size)
for _, attr := range f.node.ExtendedAttributes {
resp.Append(attr.Name)
}
return nil
}

func (f *file) Getxattr(ctx context.Context, req *fuse.GetxattrRequest, resp *fuse.GetxattrResponse) error {
debug.Log("Getxattr(%v, %v, %v)", f.node.Name, req.Name, req.Size)
attrval := f.node.GetExtendedAttribute(req.Name)
if attrval != nil {
resp.Xattr = attrval
return nil
}
return fuse.ErrNoXattr
}
3 changes: 3 additions & 0 deletions src/restic/fuse/link.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,5 +38,8 @@ func (l *link) Attr(ctx context.Context, a *fuse.Attr) error {
a.Atime = l.node.AccessTime
a.Ctime = l.node.ChangeTime
a.Mtime = l.node.ModTime

a.Nlink = uint32(l.node.Links)

return nil
}
17 changes: 15 additions & 2 deletions src/restic/fuse/snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,9 @@ var _ = fs.NodeStringLookuper(&SnapshotsDir{})
type SnapshotsDir struct {
repo restic.Repository
ownerIsRoot bool
paths []string
tags []string
host string

// knownSnapshots maps snapshot timestamp to the snapshot
sync.RWMutex
Expand All @@ -40,12 +43,15 @@ type SnapshotsDir struct {
}

// NewSnapshotsDir returns a new dir object for the snapshots.
func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool) *SnapshotsDir {
func NewSnapshotsDir(repo restic.Repository, ownerIsRoot bool, paths []string, tags []string, host string) *SnapshotsDir {
debug.Log("fuse mount initiated")
return &SnapshotsDir{
repo: repo,
knownSnapshots: make(map[string]SnapshotWithId),
ownerIsRoot: ownerIsRoot,
paths: paths,
tags: tags,
host: host,
knownSnapshots: make(map[string]SnapshotWithId),
processed: restic.NewIDSet(),
}
}
Expand Down Expand Up @@ -79,6 +85,13 @@ func (sn *SnapshotsDir) updateCache(ctx context.Context) error {
return err
}

// Filter snapshots we don't care for.
if (sn.host != "" && sn.host != snapshot.Hostname) ||
!snapshot.HasTags(sn.tags) ||
!snapshot.HasPaths(sn.paths) {
continue
}

timestamp := snapshot.Time.Format(time.RFC3339)
for i := 1; ; i++ {
if _, ok := sn.knownSnapshots[timestamp]; !ok {
Expand Down
57 changes: 57 additions & 0 deletions src/restic/hardlinks_index.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
package restic

import (
"sync"
)

// HardlinkKey is a composed key for finding inodes on a specific device.
type HardlinkKey struct {
Inode, Device uint64
}

// HardlinkIndex contains a list of inodes, devices these inodes are one, and associated file names.
type HardlinkIndex struct {
m sync.Mutex
Index map[HardlinkKey]string
}

// NewHardlinkIndex create a new index for hard links
func NewHardlinkIndex() *HardlinkIndex {
return &HardlinkIndex{
Index: make(map[HardlinkKey]string),
}
}

// Has checks wether the link already exist in the index.
func (idx *HardlinkIndex) Has(inode uint64, device uint64) bool {
idx.m.Lock()
defer idx.m.Unlock()
_, ok := idx.Index[HardlinkKey{inode, device}]

return ok
}

// Add adds a link to the index.
func (idx *HardlinkIndex) Add(inode uint64, device uint64, name string) {
idx.m.Lock()
defer idx.m.Unlock()
_, ok := idx.Index[HardlinkKey{inode, device}]

if !ok {
idx.Index[HardlinkKey{inode, device}] = name
}
}

// GetFilename obtains the filename from the index.
func (idx *HardlinkIndex) GetFilename(inode uint64, device uint64) string {
idx.m.Lock()
defer idx.m.Unlock()
return idx.Index[HardlinkKey{inode, device}]
}

// Remove removes a link from the index.
func (idx *HardlinkIndex) Remove(inode uint64, device uint64) {
idx.m.Lock()
defer idx.m.Unlock()
delete(idx.Index, HardlinkKey{inode, device})
}
35 changes: 35 additions & 0 deletions src/restic/hardlinks_index_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,35 @@
package restic_test

import (
"testing"

"restic"
. "restic/test"
)

// TestHardLinks contains various tests for HardlinkIndex.
func TestHardLinks(t *testing.T) {

idx := restic.NewHardlinkIndex()

idx.Add(1, 2, "inode1-file1-on-device2")
idx.Add(2, 3, "inode2-file2-on-device3")

var sresult string
sresult = idx.GetFilename(1, 2)
Equals(t, sresult, "inode1-file1-on-device2")

sresult = idx.GetFilename(2, 3)
Equals(t, sresult, "inode2-file2-on-device3")

var bresult bool
bresult = idx.Has(1, 2)
Equals(t, bresult, true)

bresult = idx.Has(1, 3)
Equals(t, bresult, false)

idx.Remove(1, 2)
bresult = idx.Has(1, 2)
Equals(t, bresult, false)
}
2 changes: 1 addition & 1 deletion src/restic/id.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func (id ID) String() string {
return hex.EncodeToString(id[:])
}

// NewRandomID retuns a randomly generated ID. When reading from rand fails,
// NewRandomID returns a randomly generated ID. When reading from rand fails,
// the function panics.
func NewRandomID() ID {
id := ID{}
Expand Down
2 changes: 1 addition & 1 deletion src/restic/list/list.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ func (l Result) PackID() restic.ID {
return l.packID
}

// Size ruturns the size of the pack.
// Size returns the size of the pack.
func (l Result) Size() int64 {
return l.size
}
Expand Down
2 changes: 1 addition & 1 deletion src/restic/lock.go
Original file line number Diff line number Diff line change
Expand Up @@ -203,7 +203,7 @@ func (l *Lock) Stale() bool {

hn, err := os.Hostname()
if err != nil {
debug.Log("unable to find current hostnanme: %v", err)
debug.Log("unable to find current hostname: %v", err)
// since we cannot find the current hostname, assume that the lock is
// not stale.
return false
Expand Down
184 changes: 157 additions & 27 deletions src/restic/node.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,31 +12,38 @@ import (

"restic/errors"

"runtime"

"bytes"
"restic/debug"
"restic/fs"
"runtime"
)

// ExtendedAttribute is a tuple storing the xattr name and value.
type ExtendedAttribute struct {
Name string `json:"name"`
Value []byte `json:"value"`
}

// Node is a file, directory or other item in a backup.
type Node struct {
Name string `json:"name"`
Type string `json:"type"`
Mode os.FileMode `json:"mode,omitempty"`
ModTime time.Time `json:"mtime,omitempty"`
AccessTime time.Time `json:"atime,omitempty"`
ChangeTime time.Time `json:"ctime,omitempty"`
UID uint32 `json:"uid"`
GID uint32 `json:"gid"`
User string `json:"user,omitempty"`
Group string `json:"group,omitempty"`
Inode uint64 `json:"inode,omitempty"`
Size uint64 `json:"size,omitempty"`
Links uint64 `json:"links,omitempty"`
LinkTarget string `json:"linktarget,omitempty"`
Device uint64 `json:"device,omitempty"`
Content IDs `json:"content"`
Subtree *ID `json:"subtree,omitempty"`
Name string `json:"name"`
Type string `json:"type"`
Mode os.FileMode `json:"mode,omitempty"`
ModTime time.Time `json:"mtime,omitempty"`
AccessTime time.Time `json:"atime,omitempty"`
ChangeTime time.Time `json:"ctime,omitempty"`
UID uint32 `json:"uid"`
GID uint32 `json:"gid"`
User string `json:"user,omitempty"`
Group string `json:"group,omitempty"`
Inode uint64 `json:"inode,omitempty"`
Size uint64 `json:"size,omitempty"`
Links uint64 `json:"links,omitempty"`
LinkTarget string `json:"linktarget,omitempty"`
ExtendedAttributes []ExtendedAttribute `json:"extended_attributes,omitempty"`
Device uint64 `json:"device,omitempty"`
Content IDs `json:"content"`
Subtree *ID `json:"subtree,omitempty"`

Error string `json:"error,omitempty"`

Expand All @@ -56,7 +63,8 @@ func (node Node) String() string {
return fmt.Sprintf("<Node(%s) %s>", node.Type, node.Name)
}

// NodeFromFileInfo returns a new node from the given path and FileInfo.
// NodeFromFileInfo returns a new node from the given path and FileInfo. It
// returns the first error that is encountered, together with a node.
func NodeFromFileInfo(path string, fi os.FileInfo) (*Node, error) {
mask := os.ModePerm | os.ModeType | os.ModeSetuid | os.ModeSetgid | os.ModeSticky
node := &Node{
Expand Down Expand Up @@ -96,8 +104,18 @@ func nodeTypeFromFileInfo(fi os.FileInfo) string {
return ""
}

// GetExtendedAttribute gets the extended attribute.
func (node Node) GetExtendedAttribute(a string) []byte {
for _, attr := range node.ExtendedAttributes {
if attr.Name == a {
return attr.Value
}
}
return nil
}

// CreateAt creates the node at the given path and restores all the meta data.
func (node *Node) CreateAt(path string, repo Repository) error {
func (node *Node) CreateAt(path string, repo Repository, idx *HardlinkIndex) error {
debug.Log("create node %v at %v", node.Name, path)

switch node.Type {
Expand All @@ -106,7 +124,7 @@ func (node *Node) CreateAt(path string, repo Repository) error {
return err
}
case "file":
if err := node.createFileAt(path, repo); err != nil {
if err := node.createFileAt(path, repo, idx); err != nil {
return err
}
case "symlink":
Expand Down Expand Up @@ -162,6 +180,22 @@ func (node Node) restoreMetadata(path string) error {
}
}

err = node.restoreExtendedAttributes(path)
if err != nil {
debug.Log("error restoring extended attributes for %v: %v", path, err)
return err
}

return nil
}

func (node Node) restoreExtendedAttributes(path string) error {
for _, attr := range node.ExtendedAttributes {
err := Setxattr(path, attr.Name, attr.Value)
if err != nil {
return err
}
}
return nil
}

Expand Down Expand Up @@ -191,7 +225,15 @@ func (node Node) createDirAt(path string) error {
return nil
}

func (node Node) createFileAt(path string, repo Repository) error {
func (node Node) createFileAt(path string, repo Repository, idx *HardlinkIndex) error {
if node.Links > 1 && idx.Has(node.Inode, node.Device) {
err := fs.Link(idx.GetFilename(node.Inode, node.Device), path)
if err != nil {
return errors.Wrap(err, "CreateHardlink")
}
return nil
}

f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
defer f.Close()

Expand All @@ -207,7 +249,7 @@ func (node Node) createFileAt(path string, repo Repository) error {
}

buf = buf[:cap(buf)]
if uint(len(buf)) < size {
if len(buf) < CiphertextLength(int(size)) {
buf = NewBlobBuffer(int(size))
}

Expand All @@ -223,6 +265,8 @@ func (node Node) createFileAt(path string, repo Repository) error {
}
}

idx.Add(node.Inode, node.Device, path)

return nil
}

Expand Down Expand Up @@ -340,6 +384,9 @@ func (node Node) Equals(other Node) bool {
if !node.sameContent(other) {
return false
}
if !node.sameExtendedAttributes(other) {
return false
}
if node.Subtree != nil {
if other.Subtree == nil {
return false
Expand Down Expand Up @@ -378,6 +425,51 @@ func (node Node) sameContent(other Node) bool {
return false
}
}
return true
}

func (node Node) sameExtendedAttributes(other Node) bool {
if len(node.ExtendedAttributes) != len(other.ExtendedAttributes) {
return false
}

// build a set of all attributes that node has
type mapvalue struct {
value []byte
present bool
}
attributes := make(map[string]mapvalue)
for _, attr := range node.ExtendedAttributes {
attributes[attr.Name] = mapvalue{value: attr.Value}
}

for _, attr := range other.ExtendedAttributes {
v, ok := attributes[attr.Name]
if !ok {
// extended attribute is not set for node
debug.Log("other node has attribute %v, which is not present in node", attr.Name)
return false

}

if !bytes.Equal(v.value, attr.Value) {
// attribute has different value
debug.Log("attribute %v has different value", attr.Name)
return false
}

// remember that this attribute is present in other.
v.present = true
attributes[attr.Name] = v
}

// check for attributes that are not present in other
for name, v := range attributes {
if !v.present {
debug.Log("attribute %v not present in other node", name)
return false
}
}

return true
}
Expand Down Expand Up @@ -485,18 +577,56 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error {
case "dir":
case "symlink":
node.LinkTarget, err = fs.Readlink(path)
err = errors.Wrap(err, "Readlink")
node.Links = uint64(stat.nlink())
if err != nil {
return errors.Wrap(err, "Readlink")
}
case "dev":
node.Device = uint64(stat.rdev())
node.Links = uint64(stat.nlink())
case "chardev":
node.Device = uint64(stat.rdev())
node.Links = uint64(stat.nlink())
case "fifo":
case "socket":
default:
err = errors.Errorf("invalid node type %q", node.Type)
return errors.Errorf("invalid node type %q", node.Type)
}

return err
if err = node.fillExtendedAttributes(path); err != nil {
return err
}

return nil
}

func (node *Node) fillExtendedAttributes(path string) error {
if node.Type == "symlink" {
return nil
}

xattrs, err := Listxattr(path)
debug.Log("fillExtendedAttributes(%v) %v %v", path, xattrs, err)
if err != nil {
return err
}

node.ExtendedAttributes = make([]ExtendedAttribute, 0, len(xattrs))
for _, attr := range xattrs {
attrVal, err := Getxattr(path, attr)
if err != nil {
fmt.Fprintf(os.Stderr, "can not obtain extended attribute %v for %v:\n", attr, path)
continue
}
attr := ExtendedAttribute{
Name: attr,
Value: attrVal,
}

node.ExtendedAttributes = append(node.ExtendedAttributes, attr)
}

return nil
}

type statT interface {
Expand Down
16 changes: 16 additions & 0 deletions src/restic/node_openbsd.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,3 +9,19 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe
func (s statUnix) atim() syscall.Timespec { return s.Atim }
func (s statUnix) mtim() syscall.Timespec { return s.Mtim }
func (s statUnix) ctim() syscall.Timespec { return s.Ctim }

// Getxattr retrieves extended attribute data associated with path.
func Getxattr(path, name string) ([]byte, error) {
return nil, nil
}

// Listxattr retrieves a list of names of extended attributes associated with the
// given path in the file system.
func Listxattr(path string) ([]string, error) {
return nil, nil
}

// Setxattr associates name and data together as an attribute of path.
func Setxattr(path, name string, data []byte) error {
return nil
}
4 changes: 3 additions & 1 deletion src/restic/node_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,11 @@ func TestNodeRestoreAt(t *testing.T) {
}
}()

idx := restic.NewHardlinkIndex()

for _, test := range nodeTests {
nodePath := filepath.Join(tempdir, test.Name)
OK(t, test.CreateAt(nodePath, nil))
OK(t, test.CreateAt(nodePath, nil, idx))

if test.Type == "symlink" && runtime.GOOS == "windows" {
continue
Expand Down
17 changes: 17 additions & 0 deletions src/restic/node_windows.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,25 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe
return nil
}

// Getxattr retrieves extended attribute data associated with path.
func Getxattr(path, name string) ([]byte, error) {
return nil, nil
}

// Listxattr retrieves a list of names of extended attributes associated with the
// given path in the file system.
func Listxattr(path string) ([]string, error) {
return nil, nil
}

// Setxattr associates name and data together as an attribute of path.
func Setxattr(path, name string, data []byte) error {
return nil
}

type statWin syscall.Win32FileAttributeData

//ToStatT call the Windows system call Win32FileAttributeData.
func toStatT(i interface{}) (statT, bool) {
if i == nil {
return nil, false
Expand Down
39 changes: 39 additions & 0 deletions src/restic/node_xattr.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// +build !openbsd
// +build !windows

package restic

import (
"restic/errors"
"syscall"

"github.com/pkg/xattr"
)

// Getxattr retrieves extended attribute data associated with path.
func Getxattr(path, name string) ([]byte, error) {
b, e := xattr.Getxattr(path, name)
if err, ok := e.(*xattr.XAttrError); ok && err.Err == syscall.ENOTSUP {
return nil, nil
}
return b, errors.Wrap(e, "Getxattr")
}

// Listxattr retrieves a list of names of extended attributes associated with the
// given path in the file system.
func Listxattr(path string) ([]string, error) {
s, e := xattr.Listxattr(path)
if err, ok := e.(*xattr.XAttrError); ok && err.Err == syscall.ENOTSUP {
return nil, nil
}
return s, errors.Wrap(e, "Listxattr")
}

// Setxattr associates name and data together as an attribute of path.
func Setxattr(path, name string, data []byte) error {
e := xattr.Setxattr(path, name, data)
if err, ok := e.(*xattr.XAttrError); ok && err.Err == syscall.ENOTSUP {
return nil
}
return errors.Wrap(e, "Setxattr")
}
2 changes: 1 addition & 1 deletion src/restic/repository/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -392,7 +392,7 @@ func (idx *Index) SetID(id restic.ID) error {
defer idx.m.Unlock()

if !idx.final {
return errors.New("indexs is not final")
return errors.New("index is not final")
}

if !idx.id.IsNull() {
Expand Down
66 changes: 0 additions & 66 deletions src/restic/repository/index_rebuild.go

This file was deleted.

11 changes: 8 additions & 3 deletions src/restic/repository/repack.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@ import (
// these packs. Each pack is loaded and the blobs listed in keepBlobs is saved
// into a new pack. Afterwards, the packs are removed. This operation requires
// an exclusive lock on the repo.
func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet) (err error) {
func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet, p *restic.Progress) (err error) {
debug.Log("repacking %d packs while keeping %d blobs", len(packs), len(keepBlobs))

for packID := range packs {
Expand All @@ -35,14 +35,16 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
return err
}

defer beRd.Close()

hrd := hashing.NewReader(beRd, sha256.New())
packLength, err := io.Copy(tempfile, hrd)
if err != nil {
return errors.Wrap(err, "Copy")
}

if err = beRd.Close(); err != nil {
return errors.Wrap(err, "Close")
}

hash := restic.IDFromHash(hrd.Sum(nil))
debug.Log("pack %v loaded (%d bytes), hash %v", packID.Str(), packLength, hash.Str())

Expand Down Expand Up @@ -116,6 +118,9 @@ func Repack(repo restic.Repository, packs restic.IDSet, keepBlobs restic.BlobSet
if err = os.Remove(tempfile.Name()); err != nil {
return errors.Wrap(err, "Remove")
}
if p != nil {
p.Report(restic.Stat{Blobs: 1})
}
}

if err := repo.Flush(); err != nil {
Expand Down
23 changes: 20 additions & 3 deletions src/restic/repository/repack_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"io"
"math/rand"
"restic"
"restic/index"
"restic/repository"
"testing"
)
Expand Down Expand Up @@ -131,7 +132,7 @@ func findPacksForBlobs(t *testing.T, repo restic.Repository, blobs restic.BlobSe
}

func repack(t *testing.T, repo restic.Repository, packs restic.IDSet, blobs restic.BlobSet) {
err := repository.Repack(repo, packs, blobs)
err := repository.Repack(repo, packs, blobs, nil)
if err != nil {
t.Fatal(err)
}
Expand All @@ -144,8 +145,24 @@ func saveIndex(t *testing.T, repo restic.Repository) {
}

func rebuildIndex(t *testing.T, repo restic.Repository) {
if err := repository.RebuildIndex(repo); err != nil {
t.Fatalf("error rebuilding index: %v", err)
idx, err := index.New(repo, nil)
if err != nil {
t.Fatal(err)
}

for id := range repo.List(restic.IndexFile, nil) {
err = repo.Backend().Remove(restic.Handle{
Type: restic.IndexFile,
Name: id.String(),
})
if err != nil {
t.Fatal(err)
}
}

_, err = idx.Save(repo, nil)
if err != nil {
t.Fatal(err)
}
}

Expand Down
60 changes: 16 additions & 44 deletions src/restic/repository/repository.go
Original file line number Diff line number Diff line change
Expand Up @@ -56,12 +56,12 @@ func (r *Repository) LoadAndDecrypt(t restic.FileType, id restic.ID) ([]byte, er
h := restic.Handle{Type: t, Name: id.String()}
buf, err := backend.LoadAll(r.be, h)
if err != nil {
debug.Log("error loading %v: %v", id.Str(), err)
debug.Log("error loading %v: %v", h, err)
return nil, err
}

if t != restic.ConfigFile && !restic.Hash(buf).Equal(id) {
return nil, errors.New("invalid data returned")
return nil, errors.Errorf("load %v: invalid data returned", h)
}

// decrypt
Expand Down Expand Up @@ -442,50 +442,22 @@ func (r *Repository) KeyName() string {
return r.keyName
}

func (r *Repository) list(t restic.FileType, done <-chan struct{}, out chan<- restic.ID) {
defer close(out)
in := r.be.List(t, done)

var (
// disable sending on the outCh until we received a job
outCh chan<- restic.ID
// enable receiving from in
inCh = in
id restic.ID
err error
)

for {
select {
case <-done:
return
case strID, ok := <-inCh:
if !ok {
// input channel closed, we're done
return
}
id, err = restic.ParseID(strID)
if err != nil {
// ignore invalid IDs
continue
}

inCh = nil
outCh = out
case outCh <- id:
outCh = nil
inCh = in
}
}
}

// List returns a channel that yields all IDs of type t in the backend.
func (r *Repository) List(t restic.FileType, done <-chan struct{}) <-chan restic.ID {
outCh := make(chan restic.ID)

go r.list(t, done, outCh)

return outCh
out := make(chan restic.ID)
go func() {
defer close(out)
for strID := range r.be.List(t, done) {
if id, err := restic.ParseID(strID); err == nil {
select {
case out <- id:
case <-done:
return
}
}
}
}()
return out
}

// ListPack returns the list of blobs saved in the pack id and the length of
Expand Down
19 changes: 10 additions & 9 deletions src/restic/restorer.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ func NewRestorer(repo Repository, id ID) (*Restorer, error) {
return r, nil
}

func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
func (res *Restorer) restoreTo(dst string, dir string, treeID ID, idx *HardlinkIndex) error {
tree, err := res.repo.LoadTree(treeID)
if err != nil {
return res.Error(dir, nil, err)
Expand All @@ -50,7 +50,7 @@ func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
debug.Log("SelectForRestore returned %v", selectedForRestore)

if selectedForRestore {
err := res.restoreNodeTo(node, dir, dst)
err := res.restoreNodeTo(node, dir, dst, idx)
if err != nil {
return err
}
Expand All @@ -62,7 +62,7 @@ func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
}

subp := filepath.Join(dir, node.Name)
err = res.restoreTo(dst, subp, *node.Subtree)
err = res.restoreTo(dst, subp, *node.Subtree, idx)
if err != nil {
err = res.Error(subp, node, err)
if err != nil {
Expand All @@ -83,11 +83,11 @@ func (res *Restorer) restoreTo(dst string, dir string, treeID ID) error {
return nil
}

func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string) error {
func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string, idx *HardlinkIndex) error {
debug.Log("node %v, dir %v, dst %v", node.Name, dir, dst)
dstPath := filepath.Join(dst, dir, node.Name)

err := node.CreateAt(dstPath, res.repo)
err := node.CreateAt(dstPath, res.repo, idx)
if err != nil {
debug.Log("node.CreateAt(%s) error %v", dstPath, err)
}
Expand All @@ -99,7 +99,7 @@ func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string) error {
// Create parent directories and retry
err = fs.MkdirAll(filepath.Dir(dstPath), 0700)
if err == nil || os.IsExist(errors.Cause(err)) {
err = node.CreateAt(dstPath, res.repo)
err = node.CreateAt(dstPath, res.repo, idx)
}
}

Expand All @@ -116,10 +116,11 @@ func (res *Restorer) restoreNodeTo(node *Node, dir string, dst string) error {
return nil
}

// RestoreTo creates the directories and files in the snapshot below dir.
// RestoreTo creates the directories and files in the snapshot below dst.
// Before an item is created, res.Filter is called.
func (res *Restorer) RestoreTo(dir string) error {
return res.restoreTo(dir, "", *res.sn.Tree)
func (res *Restorer) RestoreTo(dst string) error {
idx := NewHardlinkIndex()
return res.restoreTo(dst, string(filepath.Separator), *res.sn.Tree, idx)
}

// Snapshot returns the snapshot this restorer is configured to use.
Expand Down
98 changes: 65 additions & 33 deletions src/restic/snapshot.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package restic

import (
"fmt"
"os"
"os/user"
"path/filepath"
"time"
Expand All @@ -22,31 +21,28 @@ type Snapshot struct {
GID uint32 `json:"gid,omitempty"`
Excludes []string `json:"excludes,omitempty"`
Tags []string `json:"tags,omitempty"`
Original *ID `json:"original,omitempty"`

id *ID // plaintext ID, used during restore
}

// NewSnapshot returns an initialized snapshot struct for the current user and
// time.
func NewSnapshot(paths []string, tags []string) (*Snapshot, error) {
func NewSnapshot(paths []string, tags []string, hostname string) (*Snapshot, error) {
for i, path := range paths {
if p, err := filepath.Abs(path); err != nil {
paths[i] = p
}
}

sn := &Snapshot{
Paths: paths,
Time: time.Now(),
Tags: tags,
Paths: paths,
Time: time.Now(),
Tags: tags,
Hostname: hostname,
}

hn, err := os.Hostname()
if err == nil {
sn.Hostname = hn
}

err = sn.fillUserInfo()
err := sn.fillUserInfo()
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -78,16 +74,15 @@ func LoadAllSnapshots(repo Repository) (snapshots []*Snapshot, err error) {

snapshots = append(snapshots, sn)
}

return snapshots, nil
return
}

func (sn Snapshot) String() string {
return fmt.Sprintf("<Snapshot %s of %v at %s by %s@%s>",
sn.id.Str(), sn.Paths, sn.Time, sn.Username, sn.Hostname)
}

// ID retuns the snapshot's ID.
// ID returns the snapshot's ID.
func (sn Snapshot) ID() *ID {
return sn.id
}
Expand All @@ -104,7 +99,42 @@ func (sn *Snapshot) fillUserInfo() error {
return err
}

// HasTags returns true if the snapshot has all the tags.
// AddTags adds the given tags to the snapshots tags, preventing duplicates.
// It returns true if any changes were made.
func (sn *Snapshot) AddTags(addTags []string) (changed bool) {
nextTag:
for _, add := range addTags {
for _, tag := range sn.Tags {
if tag == add {
continue nextTag
}
}
sn.Tags = append(sn.Tags, add)
changed = true
}
return
}

// RemoveTags removes the given tags from the snapshots tags and
// returns true if any changes were made.
func (sn *Snapshot) RemoveTags(removeTags []string) (changed bool) {
for _, remove := range removeTags {
for i, tag := range sn.Tags {
if tag == remove {
// https://github.com/golang/go/wiki/SliceTricks
sn.Tags[i] = sn.Tags[len(sn.Tags)-1]
sn.Tags[len(sn.Tags)-1] = ""
sn.Tags = sn.Tags[:len(sn.Tags)-1]

changed = true
break
}
}
}
return
}

// HasTags returns true if the snapshot has at least all of tags.
func (sn *Snapshot) HasTags(tags []string) bool {
nextTag:
for _, tag := range tags {
Expand All @@ -120,33 +150,35 @@ nextTag:
return true
}

// SamePaths compares the Snapshot's paths and provided paths are exactly the same
func SamePaths(expected, actual []string) bool {
if len(expected) == 0 || len(actual) == 0 {
return true
}

for i := range expected {
found := false
for j := range actual {
if expected[i] == actual[j] {
found = true
break
// HasPaths returns true if the snapshot has at least all of paths.
func (sn *Snapshot) HasPaths(paths []string) bool {
nextPath:
for _, path := range paths {
for _, snPath := range sn.Paths {
if path == snPath {
continue nextPath
}
}
if !found {
return false
}

return false
}

return true
}

// SamePaths returns true if the snapshot matches the entire paths set
func (sn *Snapshot) SamePaths(paths []string) bool {
if len(sn.Paths) != len(paths) {
return false
}
return sn.HasPaths(paths)
}

// ErrNoSnapshotFound is returned when no snapshot for the given criteria could be found.
var ErrNoSnapshotFound = errors.New("no snapshot found")

// FindLatestSnapshot finds latest snapshot with optional target/directory and hostname filters.
func FindLatestSnapshot(repo Repository, targets []string, hostname string) (ID, error) {
// FindLatestSnapshot finds latest snapshot with optional target/directory, tags and hostname filters.
func FindLatestSnapshot(repo Repository, targets []string, tags []string, hostname string) (ID, error) {
var (
latest time.Time
latestID ID
Expand All @@ -158,7 +190,7 @@ func FindLatestSnapshot(repo Repository, targets []string, hostname string) (ID,
if err != nil {
return ID{}, errors.Errorf("Error listing snapshot: %v", err)
}
if snapshot.Time.After(latest) && SamePaths(snapshot.Paths, targets) && (hostname == "" || hostname == snapshot.Hostname) {
if snapshot.Time.After(latest) && (hostname == "" || hostname == snapshot.Hostname) && snapshot.HasTags(tags) && snapshot.HasPaths(targets) {
latest = snapshot.Time
latestID = snapshotID
found = true
Expand Down
2 changes: 1 addition & 1 deletion src/restic/snapshot_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,6 @@ import (
func TestNewSnapshot(t *testing.T) {
paths := []string{"/home/foobar"}

_, err := restic.NewSnapshot(paths, nil)
_, err := restic.NewSnapshot(paths, nil, "foo")
OK(t, err)
}
2 changes: 1 addition & 1 deletion src/restic/testing.go
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ func TestCreateSnapshot(t testing.TB, repo Repository, at time.Time, depth int,
t.Logf("create fake snapshot at %s with seed %d", at, seed)

fakedir := fmt.Sprintf("fakedir-at-%v", at.Format("2006-01-02 15:04:05"))
snapshot, err := NewSnapshot([]string{fakedir}, []string{"test"})
snapshot, err := NewSnapshot([]string{fakedir}, []string{"test"}, "foo")
if err != nil {
t.Fatal(err)
}
Expand Down
2 changes: 1 addition & 1 deletion src/restic/tree_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ func TestNodeComparison(t *testing.T) {
fi, err := os.Lstat("tree_test.go")
OK(t, err)

node, err := restic.NodeFromFileInfo("foo", fi)
node, err := restic.NodeFromFileInfo("tree_test.go", fi)
OK(t, err)

n2 := *node
Expand Down
2 changes: 1 addition & 1 deletion src/restic/walk/walk_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,7 @@ func TestWalkTree(t *testing.T) {

// archive a few files
arch := archiver.New(repo)
sn, _, err := arch.Snapshot(nil, dirs, nil, nil)
sn, _, err := arch.Snapshot(nil, dirs, nil, "localhost", nil)
OK(t, err)

// flush repo, write all packs
Expand Down
6 changes: 6 additions & 0 deletions vendor/manifest
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,12 @@
"revision": "8197a2e580736b78d704be0fc47b2324c0591a32",
"branch": "master"
},
{
"importpath": "github.com/pkg/xattr",
"repository": "https://github.com/pkg/xattr",
"revision": "b867675798fa7708a444945602b452ca493f2272",
"branch": "master"
},
{
"importpath": "github.com/restic/chunker",
"repository": "https://github.com/restic/chunker",
Expand Down
25 changes: 25 additions & 0 deletions vendor/src/github.com/pkg/xattr/LICENSE
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
Copyright (c) 2012 Dave Cheney. All rights reserved.
Copyright (c) 2014 Kuba Podgórski. All rights reserved.

Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:

* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.

THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
25 changes: 25 additions & 0 deletions vendor/src/github.com/pkg/xattr/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
[![GoDoc](https://godoc.org/github.com/pkg/xattr?status.svg)](http://godoc.org/github.com/pkg/xattr)
[![Go Report Card](https://goreportcard.com/badge/github.com/pkg/xattr)](https://goreportcard.com/report/github.com/pkg/xattr)
[![Build Status](https://travis-ci.org/pkg/xattr.svg?branch=master)](https://travis-ci.org/pkg/xattr)

xattr
=====
Extended attribute support for Go (linux + darwin + freebsd).

"Extended attributes are name:value pairs associated permanently with files and directories, similar to the environment strings associated with a process. An attribute may be defined or undefined. If it is defined, its value may be empty or non-empty." [See more...](https://en.wikipedia.org/wiki/Extended_file_attributes)


### Example
```
const path = "/tmp/myfile"
const prefix = "user."
if err := xattr.Setxattr(path, prefix+"test", []byte("test-attr-value")); err != nil {
log.Fatal(err)
}
var data []byte
data, err = xattr.Getxattr(path, prefix+"test"); err != nil {
log.Fatal(err)
}
```
39 changes: 39 additions & 0 deletions vendor/src/github.com/pkg/xattr/syscall_darwin.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
// +build darwin

package xattr

import (
"syscall"
"unsafe"
)

func getxattr(path string, name string, value *byte, size int, pos int, options int) (int, error) {

r0, _, e1 := syscall.Syscall6(syscall.SYS_GETXATTR, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(unsafe.Pointer(syscall.StringBytePtr(name))), uintptr(unsafe.Pointer(value)), uintptr(size), uintptr(pos), uintptr(options))
if e1 != syscall.Errno(0) {
return int(r0), e1
}
return int(r0), nil
}

func listxattr(path string, namebuf *byte, size int, options int) (int, error) {
r0, _, e1 := syscall.Syscall6(syscall.SYS_LISTXATTR, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(unsafe.Pointer(namebuf)), uintptr(size), uintptr(options), 0, 0)
if e1 != syscall.Errno(0) {
return int(r0), e1
}
return int(r0), nil
}

func setxattr(path string, name string, value *byte, size int, pos int, options int) error {
if _, _, e1 := syscall.Syscall6(syscall.SYS_SETXATTR, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(unsafe.Pointer(syscall.StringBytePtr(name))), uintptr(unsafe.Pointer(value)), uintptr(size), uintptr(pos), uintptr(options)); e1 != syscall.Errno(0) {
return e1
}
return nil
}

func removexattr(path string, name string, options int) error {
if _, _, e1 := syscall.Syscall(syscall.SYS_REMOVEXATTR, uintptr(unsafe.Pointer(syscall.StringBytePtr(path))), uintptr(unsafe.Pointer(syscall.StringBytePtr(name))), uintptr(options)); e1 != syscall.Errno(0) {
return e1
}
return nil
}
91 changes: 91 additions & 0 deletions vendor/src/github.com/pkg/xattr/syscall_freebsd.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,91 @@
// +build freebsd

package xattr

import (
"syscall"
"unsafe"
)

/*
ssize_t
extattr_get_file(const char *path, int attrnamespace,
const char *attrname, void *data, size_t nbytes);
ssize_t
extattr_set_file(const char *path, int attrnamespace,
const char *attrname, const void *data, size_t nbytes);
int
extattr_delete_file(const char *path, int attrnamespace,
const char *attrname);
ssize_t
extattr_list_file(const char *path, int attrnamespace, void *data,
size_t nbytes);
*/

func extattr_get_file(path string, attrnamespace int, attrname string, data *byte, nbytes int) (int, error) {
r, _, e := syscall.Syscall6(
syscall.SYS_EXTATTR_GET_FILE,
uintptr(unsafe.Pointer(syscall.StringBytePtr(path))),
uintptr(attrnamespace),
uintptr(unsafe.Pointer(syscall.StringBytePtr(attrname))),
uintptr(unsafe.Pointer(data)),
uintptr(nbytes),
0,
)
var err error
if e != 0 {
err = e
}
return int(r), err
}

func extattr_set_file(path string, attrnamespace int, attrname string, data *byte, nbytes int) (int, error) {
r, _, e := syscall.Syscall6(
syscall.SYS_EXTATTR_SET_FILE,
uintptr(unsafe.Pointer(syscall.StringBytePtr(path))),
uintptr(attrnamespace),
uintptr(unsafe.Pointer(syscall.StringBytePtr(attrname))),
uintptr(unsafe.Pointer(data)),
uintptr(nbytes),
0,
)
var err error
if e != 0 {
err = e
}
return int(r), err
}

func extattr_delete_file(path string, attrnamespace int, attrname string) error {
_, _, e := syscall.Syscall(
syscall.SYS_EXTATTR_DELETE_FILE,
uintptr(unsafe.Pointer(syscall.StringBytePtr(path))),
uintptr(attrnamespace),
uintptr(unsafe.Pointer(syscall.StringBytePtr(attrname))),
)
var err error
if e != 0 {
err = e
}
return err
}

func extattr_list_file(path string, attrnamespace int, data *byte, nbytes int) (int, error) {
r, _, e := syscall.Syscall6(
syscall.SYS_EXTATTR_LIST_FILE,
uintptr(unsafe.Pointer(syscall.StringBytePtr(path))),
uintptr(attrnamespace),
uintptr(unsafe.Pointer(data)),
uintptr(nbytes),
0,
0,
)
var err error
if e != 0 {
err = e
}
return int(r), err
}
32 changes: 32 additions & 0 deletions vendor/src/github.com/pkg/xattr/xattr.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
/*
Package xattr provides support for extended attributes on linux, darwin and freebsd.
Extended attributes are name:value pairs associated permanently with files and directories,
similar to the environment strings associated with a process.
An attribute may be defined or undefined. If it is defined, its value may be empty or non-empty.
More details you can find here: https://en.wikipedia.org/wiki/Extended_file_attributes
*/
package xattr

// XAttrError records an error and the operation, file path and attribute that caused it.
type XAttrError struct {
Op string
Path string
Name string
Err error
}

func (e *XAttrError) Error() string {
return e.Op + " " + e.Path + " " + e.Name + ": " + e.Err.Error()
}

// nullTermToStrings converts an array of NULL terminated UTF-8 strings to a []string.
func nullTermToStrings(buf []byte) (result []string) {
offset := 0
for index, b := range buf {
if b == 0 {
result = append(result, string(buf[offset:index]))
offset = index + 1
}
}
return
}
59 changes: 59 additions & 0 deletions vendor/src/github.com/pkg/xattr/xattr_darwin.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// +build darwin

package xattr

// Getxattr retrieves extended attribute data associated with path.
func Getxattr(path, name string) ([]byte, error) {
// find size.
size, err := getxattr(path, name, nil, 0, 0, 0)
if err != nil {
return nil, &XAttrError{"getxattr", path, name, err}
}
if size > 0 {
buf := make([]byte, size)
// Read into buffer of that size.
read, err := getxattr(path, name, &buf[0], size, 0, 0)
if err != nil {
return nil, &XAttrError{"getxattr", path, name, err}
}
return buf[:read], nil
}
return []byte{}, nil
}

// Listxattr retrieves a list of names of extended attributes associated
// with the given path in the file system.
func Listxattr(path string) ([]string, error) {
// find size.
size, err := listxattr(path, nil, 0, 0)
if err != nil {
return nil, &XAttrError{"listxattr", path, "", err}
}
if size > 0 {

buf := make([]byte, size)
// Read into buffer of that size.
read, err := listxattr(path, &buf[0], size, 0)
if err != nil {
return nil, &XAttrError{"listxattr", path, "", err}
}
return nullTermToStrings(buf[:read]), nil
}
return []string{}, nil
}

// Setxattr associates name and data together as an attribute of path.
func Setxattr(path, name string, data []byte) error {
if err := setxattr(path, name, &data[0], len(data), 0, 0); err != nil {
return &XAttrError{"setxattr", path, name, err}
}
return nil
}

// Removexattr removes the attribute associated with the given path.
func Removexattr(path, name string) error {
if err := removexattr(path, name, 0); err != nil {
return &XAttrError{"removexattr", path, name, err}
}
return nil
}
85 changes: 85 additions & 0 deletions vendor/src/github.com/pkg/xattr/xattr_freebsd.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
// +build freebsd

package xattr

import (
"syscall"
)

const (
EXTATTR_NAMESPACE_USER = 1
)

// Getxattr retrieves extended attribute data associated with path.
func Getxattr(path, name string) ([]byte, error) {
// find size.
size, err := extattr_get_file(path, EXTATTR_NAMESPACE_USER, name, nil, 0)
if err != nil {
return nil, &XAttrError{"extattr_get_file", path, name, err}
}
if size > 0 {
buf := make([]byte, size)
// Read into buffer of that size.
read, err := extattr_get_file(path, EXTATTR_NAMESPACE_USER, name, &buf[0], size)
if err != nil {
return nil, &XAttrError{"extattr_get_file", path, name, err}
}
return buf[:read], nil
}
return []byte{}, nil
}

// Listxattr retrieves a list of names of extended attributes associated
// with the given path in the file system.
func Listxattr(path string) ([]string, error) {
// find size.
size, err := extattr_list_file(path, EXTATTR_NAMESPACE_USER, nil, 0)
if err != nil {
return nil, &XAttrError{"extattr_list_file", path, "", err}
}
if size > 0 {
buf := make([]byte, size)
// Read into buffer of that size.
read, err := extattr_list_file(path, EXTATTR_NAMESPACE_USER, &buf[0], size)
if err != nil {
return nil, &XAttrError{"extattr_list_file", path, "", err}
}
return attrListToStrings(buf[:read]), nil
}
return []string{}, nil
}

// Setxattr associates name and data together as an attribute of path.
func Setxattr(path, name string, data []byte) error {
written, err := extattr_set_file(path, EXTATTR_NAMESPACE_USER, name, &data[0], len(data))
if err != nil {
return &XAttrError{"extattr_set_file", path, name, err}
}
if written != len(data) {
return &XAttrError{"extattr_set_file", path, name, syscall.E2BIG}
}
return nil
}

// Removexattr removes the attribute associated with the given path.
func Removexattr(path, name string) error {
if err := extattr_delete_file(path, EXTATTR_NAMESPACE_USER, name); err != nil {
return &XAttrError{"extattr_delete_file", path, name, err}
}
return nil
}

// attrListToStrings converts a sequnce of attribute name entries to a []string.
// Each entry consists of a single byte containing the length
// of the attribute name, followed by the attribute name.
// The name is _not_ terminated by NUL.
func attrListToStrings(buf []byte) []string {
var result []string
index := 0
for index < len(buf) {
next := index + 1 + int(buf[index])
result = append(result, string(buf[index+1:next]))
index = next
}
return result
}
61 changes: 61 additions & 0 deletions vendor/src/github.com/pkg/xattr/xattr_linux.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
// +build linux

package xattr

import "syscall"

// Getxattr retrieves extended attribute data associated with path.
func Getxattr(path, name string) ([]byte, error) {
// find size.
size, err := syscall.Getxattr(path, name, nil)
if err != nil {
return nil, &XAttrError{"getxattr", path, name, err}
}
if size > 0 {
data := make([]byte, size)
// Read into buffer of that size.
read, err := syscall.Getxattr(path, name, data)
if err != nil {
return nil, &XAttrError{"getxattr", path, name, err}
}
return data[:read], nil
}
return []byte{}, nil
}

// Listxattr retrieves a list of names of extended attributes associated
// with the given path in the file system.
func Listxattr(path string) ([]string, error) {
// find size.
size, err := syscall.Listxattr(path, nil)
if err != nil {
return nil, &XAttrError{"listxattr", path, "", err}
}
if size > 0 {
buf := make([]byte, size)
// Read into buffer of that size.
read, err := syscall.Listxattr(path, buf)
if err != nil {
return nil, &XAttrError{"listxattr", path, "", err}
}
return nullTermToStrings(buf[:read]), nil
}
return []string{}, nil
}

// Setxattr associates name and data together as an attribute of path.
func Setxattr(path, name string, data []byte) error {
if err := syscall.Setxattr(path, name, data, 0); err != nil {
return &XAttrError{"setxattr", path, name, err}
}
return nil
}

// Removexattr removes the attribute associated
// with the given path.
func Removexattr(path, name string) error {
if err := syscall.Removexattr(path, name); err != nil {
return &XAttrError{"removexattr", path, name, err}
}
return nil
}
57 changes: 57 additions & 0 deletions vendor/src/github.com/pkg/xattr/xattr_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
// +build linux darwin freebsd

package xattr

import (
"io/ioutil"
"os"
"testing"
)

const UserPrefix = "user."

func Test_setxattr(t *testing.T) {
tmp, err := ioutil.TempFile("", "")

if err != nil {
t.Fatal(err)
}
defer os.Remove(tmp.Name())

err = Setxattr(tmp.Name(), UserPrefix+"test", []byte("test-attr-value"))
if err != nil {
t.Fatal(err)
}

list, err := Listxattr(tmp.Name())
if err != nil {
t.Fatal(err)
}

found := false
for _, name := range list {
if name == UserPrefix+"test" {
found = true
}
}

if !found {
t.Fatal("Listxattr did not return test attribute")
}

var data []byte
data, err = Getxattr(tmp.Name(), UserPrefix+"test")
if err != nil {
t.Fatal(err)
}
value := string(data)
t.Log(value)
if "test-attr-value" != value {
t.Fail()
}

err = Removexattr(tmp.Name(), UserPrefix+"test")
if err != nil {
t.Fatal(err)
}
}