Fix 567 (#570)

* Patch for  #567
Backup also files on windows with longer pathnames than 255 chars (e.g. from node).

as fd0 says "So, as far as I can see, we need to have custom methods for all functions that accept a path, so that on Windows we can substitute the normal (possibly relative) path used within restic by an (absolute) UNC path, and only then call the underlying functions like os.Stat(), os.Lstat(), os.Open() and so on.

I've already thought about adding a generic abstraction for the file system (so we can mock this easier in tests), and this looks like a good opportunity to build it."

* fixed building tests

* Restructured patches
Add Wrapper for filepath.Walk

* using \\?\ requires absolute pathes to be used.
Now all tests run

* used gofmt on the code

* Restructured Code. No patches dir, integrate the file functions into restic/fs/

There is still an issue, because restic.fs.Open has a different api the os.Open, which returns the result of OpenFile, but takes only a string

* Changed the last os.Open() calls to fs.Open() after extending the File interface

* fixed name-clash of restic.fs and fuse.fs detected by travis

* fixed fmt with gofmt

* c&p failure: removed fixpath() call.

* missing include

* fixed includes in linux variant

* Fix for Linux. Fd() is required on File interface

* done gofmt

src/cmds/restic-server/handlers.go

@@ -12,6 +12,8 @@ import (
 	"path/filepath"
 	"strings"
 	"time"
+
+	"restic/fs"
 )
 
 // Context contains repository meta-data.
@@ -134,7 +136,7 @@ func GetBlob(c *Context) http.HandlerFunc {
 		dir := vars[1]
 		name := vars[2]
 		path := filepath.Join(c.path, dir, name)
-		file, err := os.Open(path)
+		file, err := fs.Open(path)
 		if err != nil {
 			http.Error(w, "404 not found", 404)
 			return
@@ -152,7 +154,7 @@ func SaveBlob(c *Context) http.HandlerFunc {
 		name := vars[2]
 		path := filepath.Join(c.path, dir, name)
 		tmp := path + "_tmp"
-		tf, err := os.OpenFile(tmp, os.O_CREATE|os.O_WRONLY, 0600)
+		tf, err := fs.OpenFile(tmp, os.O_CREATE|os.O_WRONLY, 0600)
 		if err != nil {
 			http.Error(w, "500 internal server error", 500)
 			return

src/cmds/restic-server/htpasswd.go

@@ -32,7 +32,8 @@ import (
 	"encoding/csv"
 	"io"
 	"log"
-	"os"
+
+	"restic/fs"
 )
 
 // lookup passwords in a htpasswd file
@@ -47,7 +48,7 @@ type HtpasswdFile struct {
 // file and returns them. If an error is encountered, it is returned, together
 // with a nil-Pointer for the HtpasswdFile.
 func NewHtpasswdFromFile(path string) (*HtpasswdFile, error) {
-	r, err := os.Open(path)
+	r, err := fs.Open(path)
 	if err != nil {
 		return nil, err
 	}

src/cmds/restic/cmd_backup.go

@@ -10,6 +10,7 @@ import (
 	"restic/backend"
 	"restic/debug"
 	"restic/filter"
+	"restic/fs"
 	"strings"
 	"time"
 
@@ -221,7 +222,7 @@ func (cmd CmdBackup) newArchiveStdinProgress() *restic.Progress {
 // items exist at all.
 func filterExisting(items []string) (result []string, err error) {
 	for _, item := range items {
-		_, err := os.Lstat(item)
+		_, err := fs.Lstat(item)
 		if err != nil && os.IsNotExist(err) {
 			continue
 		}
@@ -334,7 +335,7 @@ func (cmd CmdBackup) Execute(args []string) error {
 
 	// add patterns from file
 	if cmd.ExcludeFile != "" {
-		file, err := os.Open(cmd.ExcludeFile)
+		file, err := fs.Open(cmd.ExcludeFile)
 		if err != nil {
 			cmd.global.Warnf("error reading exclude patterns: %v", err)
 			return nil

src/cmds/restic/cmd_mount.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"os"
 
+	resticfs "restic/fs"
 	"restic/fuse"
 
 	systemFuse "bazil.org/fuse"
@@ -55,9 +56,9 @@ func (cmd CmdMount) Execute(args []string) error {
 	}
 
 	mountpoint := args[0]
-	if _, err := os.Stat(mountpoint); os.IsNotExist(err) {
+	if _, err := resticfs.Stat(mountpoint); os.IsNotExist(err) {
 		cmd.global.Verbosef("Mountpoint %s doesn't exist, creating it\n", mountpoint)
-		err = os.Mkdir(mountpoint, os.ModeDir|0700)
+		err = resticfs.Mkdir(mountpoint, os.ModeDir|0700)
 		if err != nil {
 			return err
 		}

src/restic/archiver.go

@@ -780,7 +780,7 @@ func Scan(dirs []string, filter pipe.SelectFunc, p *Progress) (Stat, error) {
 
 	for _, dir := range dirs {
 		debug.Log("Scan", "Start for %v", dir)
-		err := filepath.Walk(dir, func(str string, fi os.FileInfo, err error) error {
+		err := fs.Walk(dir, func(str string, fi os.FileInfo, err error) error {
 			// TODO: integrate error reporting
 			if err != nil {
 				fmt.Fprintf(os.Stderr, "error for %v: %v\n", str, err)

src/restic/backend/local/local.go

@@ -34,7 +34,7 @@ func paths(dir string) []string {
 func Open(dir string) (*Local, error) {
 	// test if all necessary dirs are there
 	for _, d := range paths(dir) {
-		if _, err := os.Stat(d); err != nil {
+		if _, err := fs.Stat(d); err != nil {
 			return nil, fmt.Errorf("%s does not exist", d)
 		}
 	}
@@ -46,14 +46,14 @@ func Open(dir string) (*Local, error) {
 // backend at dir. Afterwards a new config blob should be created.
 func Create(dir string) (*Local, error) {
 	// test if config file already exists
-	_, err := os.Lstat(filepath.Join(dir, backend.Paths.Config))
+	_, err := fs.Lstat(filepath.Join(dir, backend.Paths.Config))
 	if err == nil {
 		return nil, errors.New("config file already exists")
 	}
 
 	// create paths for data, refs and temp
 	for _, d := range paths(dir) {
-		err := os.MkdirAll(d, backend.Modes.Dir)
+		err := fs.MkdirAll(d, backend.Modes.Dir)
 		if err != nil {
 			return nil, err
 		}
@@ -105,7 +105,7 @@ func (b *Local) Load(h backend.Handle, p []byte, off int64) (n int, err error) {
 		return 0, err
 	}
 
-	f, err := os.Open(filename(b.p, h.Type, h.Name))
+	f, err := fs.Open(filename(b.p, h.Type, h.Name))
 	if err != nil {
 		return 0, err
 	}
@@ -172,19 +172,19 @@ func (b *Local) Save(h backend.Handle, p []byte) (err error) {
 	filename := filename(b.p, h.Type, h.Name)
 
 	// test if new path already exists
-	if _, err := os.Stat(filename); err == nil {
+	if _, err := fs.Stat(filename); err == nil {
 		return fmt.Errorf("Rename(): file %v already exists", filename)
 	}
 
 	// create directories if necessary, ignore errors
 	if h.Type == backend.Data {
-		err = os.MkdirAll(filepath.Dir(filename), backend.Modes.Dir)
+		err = fs.MkdirAll(filepath.Dir(filename), backend.Modes.Dir)
 		if err != nil {
 			return err
 		}
 	}
 
-	err = os.Rename(tmpfile, filename)
+	err = fs.Rename(tmpfile, filename)
 	debug.Log("local.Save", "save %v: rename %v -> %v: %v",
 		h, filepath.Base(tmpfile), filepath.Base(filename), err)
 
@@ -193,7 +193,7 @@ func (b *Local) Save(h backend.Handle, p []byte) (err error) {
 	}
 
 	// set mode to read-only
-	fi, err := os.Stat(filename)
+	fi, err := fs.Stat(filename)
 	if err != nil {
 		return err
 	}
@@ -207,7 +207,7 @@ func (b *Local) Stat(h backend.Handle) (backend.BlobInfo, error) {
 		return backend.BlobInfo{}, err
 	}
 
-	fi, err := os.Stat(filename(b.p, h.Type, h.Name))
+	fi, err := fs.Stat(filename(b.p, h.Type, h.Name))
 	if err != nil {
 		return backend.BlobInfo{}, err
 	}
@@ -217,7 +217,7 @@ func (b *Local) Stat(h backend.Handle) (backend.BlobInfo, error) {
 
 // Test returns true if a blob of the given type and name exists in the backend.
 func (b *Local) Test(t backend.Type, name string) (bool, error) {
-	_, err := os.Stat(filename(b.p, t, name))
+	_, err := fs.Stat(filename(b.p, t, name))
 	if err != nil {
 		if os.IsNotExist(err) {
 			return false, nil
@@ -233,20 +233,20 @@ func (b *Local) Remove(t backend.Type, name string) error {
 	fn := filename(b.p, t, name)
 
 	// reset read-only flag
-	err := os.Chmod(fn, 0666)
+	err := fs.Chmod(fn, 0666)
 	if err != nil {
 		return err
 	}
 
-	return os.Remove(fn)
+	return fs.Remove(fn)
 }
 
 func isFile(fi os.FileInfo) bool {
 	return fi.Mode()&(os.ModeType|os.ModeCharDevice) == 0
 }
 
 func readdir(d string) (fileInfos []os.FileInfo, err error) {
-	f, e := os.Open(d)
+	f, e := fs.Open(d)
 	if e != nil {
 		return nil, e
 	}
@@ -336,7 +336,7 @@ func (b *Local) List(t backend.Type, done <-chan struct{}) <-chan string {
 
 // Delete removes the repository and all files.
 func (b *Local) Delete() error {
-	return os.RemoveAll(b.p)
+	return fs.RemoveAll(b.p)
 }
 
 // Close closes all open files.

src/restic/backend/local/local_unix.go

@@ -4,9 +4,10 @@ package local
 
 import (
 	"os"
+	"restic/fs"
 )
 
 // set file to readonly
 func setNewFileMode(f string, fi os.FileInfo) error {
-	return os.Chmod(f, fi.Mode()&os.FileMode(^uint32(0222)))
+	return fs.Chmod(f, fi.Mode()&os.FileMode(^uint32(0222)))
 }

src/restic/cache.go

@@ -11,6 +11,7 @@ import (
 
 	"restic/backend"
 	"restic/debug"
+	"restic/fs"
 	"restic/repository"
 )
 
@@ -43,7 +44,7 @@ func (c *Cache) Has(t backend.Type, subtype string, id backend.ID) (bool, error)
 	if err != nil {
 		return false, err
 	}
-	fd, err := os.Open(filename)
+	fd, err := fs.Open(filename)
 	defer fd.Close()
 
 	if err != nil {
@@ -70,12 +71,12 @@ func (c *Cache) Store(t backend.Type, subtype string, id backend.ID) (io.WriteCl
 	}
 
 	dirname := filepath.Dir(filename)
-	err = os.MkdirAll(dirname, 0700)
+	err = fs.MkdirAll(dirname, 0700)
 	if err != nil {
 		return nil, err
 	}
 
-	file, err := os.Create(filename)
+	file, err := fs.Create(filename)
 	if err != nil {
 		debug.Log("Cache.Store", "error creating file %v: %v", filename, err)
 		return nil, err
@@ -93,7 +94,7 @@ func (c *Cache) Load(t backend.Type, subtype string, id backend.ID) (io.ReadClos
 		return nil, err
 	}
 
-	return os.Open(filename)
+	return fs.Open(filename)
 }
 
 func (c *Cache) purge(t backend.Type, subtype string, id backend.ID) error {
@@ -102,7 +103,7 @@ func (c *Cache) purge(t backend.Type, subtype string, id backend.ID) error {
 		return err
 	}
 
-	err = os.Remove(filename)
+	err = fs.Remove(filename)
 	debug.Log("Cache.purge", "Remove file %v: %v", filename, err)
 
 	if err != nil && os.IsNotExist(err) {
@@ -157,7 +158,7 @@ func (c *Cache) list(t backend.Type) ([]cacheEntry, error) {
 		return nil, fmt.Errorf("cache not supported for type %v", t)
 	}
 
-	fd, err := os.Open(dir)
+	fd, err := fs.Open(dir)
 	if err != nil {
 		if os.IsNotExist(err) {
 			return []cacheEntry{}, nil
@@ -228,10 +229,10 @@ func getWindowsCacheDir() (string, error) {
 		cachedir = os.TempDir()
 	}
 	cachedir = filepath.Join(cachedir, "restic")
-	fi, err := os.Stat(cachedir)
+	fi, err := fs.Stat(cachedir)
 
 	if os.IsNotExist(err) {
-		err = os.MkdirAll(cachedir, 0700)
+		err = fs.MkdirAll(cachedir, 0700)
 		if err != nil {
 			return "", err
 		}
@@ -266,14 +267,14 @@ func getXDGCacheDir() (string, error) {
 		cachedir = filepath.Join(home, ".cache", "restic")
 	}
 
-	fi, err := os.Stat(cachedir)
+	fi, err := fs.Stat(cachedir)
 	if os.IsNotExist(err) {
-		err = os.MkdirAll(cachedir, 0700)
+		err = fs.MkdirAll(cachedir, 0700)
 		if err != nil {
 			return "", err
 		}
 
-		fi, err = os.Stat(cachedir)
+		fi, err = fs.Stat(cachedir)
 		debug.Log("getCacheDir", "create cache dir %v", cachedir)
 	}
 

src/restic/debug/debug.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"restic/fs"
 	"runtime"
 	"strconv"
 	"strings"
@@ -48,7 +49,7 @@ func initDebugLogger() {
 
 	fmt.Fprintf(os.Stderr, "debug log file %v\n", debugfile)
 
-	f, err := os.OpenFile(debugfile, os.O_WRONLY|os.O_APPEND, 0600)
+	f, err := fs.OpenFile(debugfile, os.O_WRONLY|os.O_APPEND, 0600)
 
 	if err == nil {
 		_, err = f.Seek(2, 0)
@@ -59,7 +60,7 @@ func initDebugLogger() {
 	}
 
 	if err != nil && os.IsNotExist(err) {
-		f, err = os.OpenFile(debugfile, os.O_WRONLY|os.O_CREATE, 0600)
+		f, err = fs.OpenFile(debugfile, os.O_WRONLY|os.O_CREATE, 0600)
 	}
 
 	if err != nil {

src/restic/fs/file.go

@@ -11,5 +11,9 @@ type File interface {
 	io.Writer
 	io.Closer
 
+	Fd() uintptr
+	Readdirnames(n int) ([]string, error)
+	Readdir(int) ([]os.FileInfo, error)
+	Seek(int64, int) (int64, error)
 	Stat() (os.FileInfo, error)
 }