Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
Merge pull request #763 from jgfrm/issue25
Support hard links
  • Loading branch information
fd0 committed Feb 10, 2017
2 parents 073edd9 + 366bf4e commit 6300c8d
Show file tree
Hide file tree
Showing 12 changed files with 281 additions and 11 deletions.
2 changes: 2 additions & 0 deletions src/cmds/restic/integration_helpers_test.go
Expand Up @@ -15,6 +15,7 @@ import (
type dirEntry struct {
path string
fi os.FileInfo
link uint64
}

func walkDir(dir string) <-chan *dirEntry {
Expand All @@ -36,6 +37,7 @@ func walkDir(dir string) <-chan *dirEntry {
ch <- &dirEntry{
path: name,
fi: info,
link: nlink(info),
}

return nil
Expand Down
34 changes: 34 additions & 0 deletions src/cmds/restic/integration_helpers_unix_test.go
Expand Up @@ -4,7 +4,9 @@ package main

import (
"fmt"
"io/ioutil"
"os"
"path/filepath"
"syscall"
)

Expand Down Expand Up @@ -37,5 +39,37 @@ func (e *dirEntry) equals(other *dirEntry) bool {
return false
}

if stat.Nlink != stat2.Nlink {
fmt.Fprintf(os.Stderr, "%v: Number of links do not match (%v != %v)\n", e.path, stat.Nlink, stat2.Nlink)
return false
}

return true
}

func nlink(info os.FileInfo) uint64 {
stat, _ := info.Sys().(*syscall.Stat_t)
return uint64(stat.Nlink)
}

func inode(info os.FileInfo) uint64 {
stat, _ := info.Sys().(*syscall.Stat_t)
return uint64(stat.Ino)
}

func createFileSetPerHardlink(dir string) map[uint64][]string {
var stat syscall.Stat_t
linkTests := make(map[uint64][]string)
files, err := ioutil.ReadDir(dir)
if err != nil {
return nil
}
for _, f := range files {

if err := syscall.Stat(filepath.Join(dir, f.Name()), &stat); err != nil {
return nil
}
linkTests[uint64(stat.Ino)] = append(linkTests[uint64(stat.Ino)], f.Name())
}
return linkTests
}
22 changes: 22 additions & 0 deletions src/cmds/restic/integration_helpers_windows_test.go
Expand Up @@ -4,6 +4,7 @@ package main

import (
"fmt"
"io/ioutil"
"os"
)

Expand All @@ -25,3 +26,24 @@ func (e *dirEntry) equals(other *dirEntry) bool {

return true
}

func nlink(info os.FileInfo) uint64 {
return 1
}

func inode(info os.FileInfo) uint64 {
return uint64(0)
}

func createFileSetPerHardlink(dir string) map[uint64][]string {
linkTests := make(map[uint64][]string)
files, err := ioutil.ReadDir(dir)
if err != nil {
return nil
}
for i, f := range files {
linkTests[uint64(i)] = append(linkTests[uint64(i)], f.Name())
i++
}
return linkTests
}
97 changes: 97 additions & 0 deletions src/cmds/restic/integration_test.go
Expand Up @@ -1011,3 +1011,100 @@ func TestPrune(t *testing.T) {
testRunCheck(t, gopts)
})
}

func TestHardLink(t *testing.T) {
// this test assumes a test set with a single directory containing hard linked files
withTestEnvironment(t, func(env *testEnvironment, gopts GlobalOptions) {
datafile := filepath.Join("testdata", "test.hl.tar.gz")
fd, err := os.Open(datafile)
if os.IsNotExist(errors.Cause(err)) {
t.Skipf("unable to find data file %q, skipping", datafile)
return
}
OK(t, err)
OK(t, fd.Close())

testRunInit(t, gopts)

SetupTarTestFixture(t, env.testdata, datafile)

linkTests := createFileSetPerHardlink(env.testdata)

opts := BackupOptions{}

// first backup
testRunBackup(t, []string{env.testdata}, opts, gopts)
snapshotIDs := testRunList(t, "snapshots", gopts)
Assert(t, len(snapshotIDs) == 1,
"expected one snapshot, got %v", snapshotIDs)

testRunCheck(t, gopts)

// restore all backups and compare
for i, snapshotID := range snapshotIDs {
restoredir := filepath.Join(env.base, fmt.Sprintf("restore%d", i))
t.Logf("restoring snapshot %v to %v", snapshotID.Str(), restoredir)
testRunRestore(t, gopts, restoredir, snapshotIDs[0])
Assert(t, directoriesEqualContents(env.testdata, filepath.Join(restoredir, "testdata")),
"directories are not equal")

linkResults := createFileSetPerHardlink(filepath.Join(restoredir, "testdata"))
Assert(t, linksEqual(linkTests, linkResults),
"links are not equal")
}

testRunCheck(t, gopts)
})
}

func linksEqual(source, dest map[uint64][]string) bool {
for _, vs := range source {
found := false
for kd, vd := range dest {
if linkEqual(vs, vd) {
delete(dest, kd)
found = true
break
}
}
if !found {
return false
}
}

if len(dest) != 0 {
return false
}

return true
}

func linkEqual(source, dest []string) bool {
// equal if sliced are equal without considering order
if source == nil && dest == nil {
return true
}

if source == nil || dest == nil {
return false
}

if len(source) != len(dest) {
return false
}

for i := range source {
found := false
for j := range dest {
if source[i] == dest[j] {
found = true
break
}
}
if !found {
return false
}
}

return true
}
Binary file added src/cmds/restic/testdata/test.hl.tar.gz
Binary file not shown.
6 changes: 6 additions & 0 deletions src/restic/fs/file.go
Expand Up @@ -102,6 +102,12 @@ func Symlink(oldname, newname string) error {
return os.Symlink(fixpath(oldname), fixpath(newname))
}

// Link creates newname as a hard link to oldname.
// If there is an error, it will be of type *LinkError.
func Link(oldname, newname string) error {
return os.Link(fixpath(oldname), fixpath(newname))
}

// Stat returns a FileInfo structure describing the named file.
// If there is an error, it will be of type *PathError.
func Stat(name string) (os.FileInfo, error) {
Expand Down
57 changes: 57 additions & 0 deletions src/restic/hardlinks_index.go
@@ -0,0 +1,57 @@
package restic

import (
"sync"
)

// HardlinkKey is a composed key for finding inodes on a specific device.
type HardlinkKey struct {
Inode, Device uint64
}

// HardlinkIndex contains a list of inodes, devices these inodes are one, and associated file names.
type HardlinkIndex struct {
m sync.Mutex
Index map[HardlinkKey]string
}

// NewHardlinkIndex create a new index for hard links
func NewHardlinkIndex() *HardlinkIndex {
return &HardlinkIndex{
Index: make(map[HardlinkKey]string),
}
}

// Has checks wether the link already exist in the index.
func (idx *HardlinkIndex) Has(inode uint64, device uint64) bool {
idx.m.Lock()
defer idx.m.Unlock()
_, ok := idx.Index[HardlinkKey{inode, device}]

return ok
}

// Add adds a link to the index.
func (idx *HardlinkIndex) Add(inode uint64, device uint64, name string) {
idx.m.Lock()
defer idx.m.Unlock()
_, ok := idx.Index[HardlinkKey{inode, device}]

if !ok {
idx.Index[HardlinkKey{inode, device}] = name
}
}

// GetFilename obtains the filename from the index.
func (idx *HardlinkIndex) GetFilename(inode uint64, device uint64) string {
idx.m.Lock()
defer idx.m.Unlock()
return idx.Index[HardlinkKey{inode, device}]
}

// Remove removes a link from the index.
func (idx *HardlinkIndex) Remove(inode uint64, device uint64) {
idx.m.Lock()
defer idx.m.Unlock()
delete(idx.Index, HardlinkKey{inode, device})
}
35 changes: 35 additions & 0 deletions src/restic/hardlinks_index_test.go
@@ -0,0 +1,35 @@
package restic_test

import (
"testing"

"restic"
. "restic/test"
)

// TestHardLinks contains various tests for HardlinkIndex.
func TestHardLinks(t *testing.T) {

idx := restic.NewHardlinkIndex()

idx.Add(1, 2, "inode1-file1-on-device2")
idx.Add(2, 3, "inode2-file2-on-device3")

var sresult string
sresult = idx.GetFilename(1, 2)
Equals(t, sresult, "inode1-file1-on-device2")

sresult = idx.GetFilename(2, 3)
Equals(t, sresult, "inode2-file2-on-device3")

var bresult bool
bresult = idx.Has(1, 2)
Equals(t, bresult, true)

bresult = idx.Has(1, 3)
Equals(t, bresult, false)

idx.Remove(1, 2)
bresult = idx.Has(1, 2)
Equals(t, bresult, false)
}
19 changes: 16 additions & 3 deletions src/restic/node.go
Expand Up @@ -97,7 +97,7 @@ func nodeTypeFromFileInfo(fi os.FileInfo) string {
}

// CreateAt creates the node at the given path and restores all the meta data.
func (node *Node) CreateAt(path string, repo Repository) error {
func (node *Node) CreateAt(path string, repo Repository, idx *HardlinkIndex) error {
debug.Log("create node %v at %v", node.Name, path)

switch node.Type {
Expand All @@ -106,7 +106,7 @@ func (node *Node) CreateAt(path string, repo Repository) error {
return err
}
case "file":
if err := node.createFileAt(path, repo); err != nil {
if err := node.createFileAt(path, repo, idx); err != nil {
return err
}
case "symlink":
Expand Down Expand Up @@ -191,7 +191,15 @@ func (node Node) createDirAt(path string) error {
return nil
}

func (node Node) createFileAt(path string, repo Repository) error {
func (node Node) createFileAt(path string, repo Repository, idx *HardlinkIndex) error {
if node.Links > 1 && idx.Has(node.Inode, node.Device) {
err := fs.Link(idx.GetFilename(node.Inode, node.Device), path)
if err != nil {
return errors.Wrap(err, "CreateHardlink")
}
return nil
}

f, err := fs.OpenFile(path, os.O_CREATE|os.O_WRONLY, 0600)
defer f.Close()

Expand Down Expand Up @@ -223,6 +231,8 @@ func (node Node) createFileAt(path string, repo Repository) error {
}
}

idx.Add(node.Inode, node.Device, path)

return nil
}

Expand Down Expand Up @@ -485,11 +495,14 @@ func (node *Node) fillExtra(path string, fi os.FileInfo) error {
case "dir":
case "symlink":
node.LinkTarget, err = fs.Readlink(path)
node.Links = uint64(stat.nlink())
err = errors.Wrap(err, "Readlink")
case "dev":
node.Device = uint64(stat.rdev())
node.Links = uint64(stat.nlink())
case "chardev":
node.Device = uint64(stat.rdev())
node.Links = uint64(stat.nlink())
case "fifo":
case "socket":
default:
Expand Down
4 changes: 3 additions & 1 deletion src/restic/node_test.go
Expand Up @@ -176,9 +176,11 @@ func TestNodeRestoreAt(t *testing.T) {
}
}()

idx := restic.NewHardlinkIndex()

for _, test := range nodeTests {
nodePath := filepath.Join(tempdir, test.Name)
OK(t, test.CreateAt(nodePath, nil))
OK(t, test.CreateAt(nodePath, nil, idx))

if test.Type == "symlink" && runtime.GOOS == "windows" {
continue
Expand Down
1 change: 1 addition & 0 deletions src/restic/node_windows.go
Expand Up @@ -24,6 +24,7 @@ func (node Node) restoreSymlinkTimestamps(path string, utimes [2]syscall.Timespe

type statWin syscall.Win32FileAttributeData

//ToStatT call the Windows system call Win32FileAttributeData.
func toStatT(i interface{}) (statT, bool) {
if i == nil {
return nil, false
Expand Down

0 comments on commit 6300c8d

Please sign in to comment.