sync.go

package drive

import (
	"fmt"
	"github.com/sabhiram/go-git-ignore"
	"github.com/soniakeys/graph"
	"google.golang.org/api/drive/v3"
	"google.golang.org/api/googleapi"
	"io"
	"os"
	"path/filepath"
	"strings"
	"text/tabwriter"
	"time"
)

const DefaultIgnoreFile = ".gdriveignore"

type ModTime int

const (
	LocalLastModified ModTime = iota
	RemoteLastModified
	EqualModifiedTime
)

type LargestSize int

const (
	LocalLargestSize LargestSize = iota
	RemoteLargestSize
	EqualSize
)

type ConflictResolution int

const (
	NoResolution ConflictResolution = iota
	KeepLocal
	KeepRemote
	KeepLargest
)

func (self *Drive) prepareSyncFiles(localPath string, root *drive.File, cmp FileComparer) (*syncFiles, error) {
	localCh := make(chan struct {
		files []*LocalFile
		err   error
	})
	remoteCh := make(chan struct {
		files []*RemoteFile
		err   error
	})

	go func() {
		files, err := prepareLocalFiles(localPath)
		localCh <- struct {
			files []*LocalFile
			err   error
		}{files, err}
	}()

	go func() {
		files, err := self.prepareRemoteFiles(root, "")
		remoteCh <- struct {
			files []*RemoteFile
			err   error
		}{files, err}
	}()

	local := <-localCh
	if local.err != nil {
		return nil, local.err
	}

	remote := <-remoteCh
	if remote.err != nil {
		return nil, remote.err
	}

	return &syncFiles{
		root:    &RemoteFile{file: root},
		local:   local.files,
		remote:  remote.files,
		compare: cmp,
	}, nil
}

func (self *Drive) isSyncFile(id string) (bool, error) {
	f, err := self.service.Files.Get(id).Fields("appProperties").Do()
	if err != nil {
		return false, fmt.Errorf("Failed to get file: %s", err)
	}

	_, ok := f.AppProperties["sync"]
	return ok, nil
}

func prepareLocalFiles(root string) ([]*LocalFile, error) {
	var files []*LocalFile

	// Get absolute root path
	absRootPath, err := filepath.Abs(root)
	if err != nil {
		return nil, err
	}

	// Prepare ignorer
	shouldIgnore, err := prepareIgnorer(filepath.Join(absRootPath, DefaultIgnoreFile))
	if err != nil {
		return nil, err
	}

	err = filepath.Walk(absRootPath, func(absPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Skip root directory
		if absPath == absRootPath {
			return nil
		}

		// Skip files that are not a directory or regular file
		if !info.IsDir() && !info.Mode().IsRegular() {
			return nil
		}

		// Get relative path from root
		relPath, err := filepath.Rel(absRootPath, absPath)
		if err != nil {
			return err
		}

		// Skip file if it is ignored by ignore file
		if shouldIgnore(relPath) {
			return nil
		}

		files = append(files, &LocalFile{
			absPath: absPath,
			relPath: relPath,
			info:    info,
		})

		return nil
	})

	if err != nil {
		return nil, fmt.Errorf("Failed to prepare local files: %s", err)
	}

	return files, err
}

func (self *Drive) prepareRemoteFiles(rootDir *drive.File, sortOrder string) ([]*RemoteFile, error) {
	// Find all files which has rootDir as root
	listArgs := listAllFilesArgs{
		query:     fmt.Sprintf("appProperties has {key='syncRootId' and value='%s'}", rootDir.Id),
		fields:    []googleapi.Field{"nextPageToken", "files(id,name,parents,md5Checksum,mimeType,size,modifiedTime)"},
		sortOrder: sortOrder,
	}
	files, err := self.listAllFiles(listArgs)
	if err != nil {
		return nil, fmt.Errorf("Failed listing files: %s", err)
	}

	if err := checkFiles(files); err != nil {
		return nil, err
	}

	relPaths, err := prepareRemoteRelPaths(rootDir, files)
	if err != nil {
		return nil, err
	}

	var remoteFiles []*RemoteFile
	for _, f := range files {
		relPath, ok := relPaths[f.Id]
		if !ok {
			return nil, fmt.Errorf("File %s does not have a valid parent", f.Id)
		}
		remoteFiles = append(remoteFiles, &RemoteFile{
			relPath: relPath,
			file:    f,
		})
	}

	return remoteFiles, nil
}

func prepareRemoteRelPaths(root *drive.File, files []*drive.File) (map[string]string, error) {
	// The tree only holds integer values so we use
	// maps to lookup file by index and index by file id
	indexLookup := map[string]graph.NI{}
	fileLookup := map[graph.NI]*drive.File{}

	// All files includes root dir
	allFiles := append([]*drive.File{root}, files...)

	// Prepare lookup maps
	for i, f := range allFiles {
		indexLookup[f.Id] = graph.NI(i)
		fileLookup[graph.NI(i)] = f
	}

	// This will hold 'parent index' -> 'file index' relationships
	pathEnds := make([]graph.PathEnd, len(allFiles))

	// Prepare parent -> file relationships
	for i, f := range allFiles {
		if f == root {
			pathEnds[i] = graph.PathEnd{From: -1}
			continue
		}

		// Lookup index of parent
		parentIdx, found := indexLookup[f.Parents[0]]
		if !found {
			return nil, fmt.Errorf("Could not find parent of %s (%s)", f.Id, f.Name)
		}
		pathEnds[i] = graph.PathEnd{From: parentIdx}
	}

	// Create parent pointer tree and calculate path lengths
	tree := &graph.FromList{Paths: pathEnds}
	tree.RecalcLeaves()
	tree.RecalcLen()

	// This will hold a map of file id => relative path
	paths := map[string]string{}

	// Find relative path from root for all files
	for _, f := range allFiles {
		if f == root {
			continue
		}

		// Find nodes between root and file
		nodes := tree.PathTo(indexLookup[f.Id], nil)

		// This will hold the name of all paths between root and
		// file (exluding root and including file itself)
		pathNames := []string{}

		// Lookup file for each node and grab name
		for _, n := range nodes {
			file := fileLookup[n]
			if file == root {
				continue
			}
			pathNames = append(pathNames, file.Name)
		}

		// Join path names to form relative path and add to map
		paths[f.Id] = filepath.Join(pathNames...)
	}

	return paths, nil
}

func checkFiles(files []*drive.File) error {
	uniq := map[string]string{}

	for _, f := range files {
		// Ensure all files have exactly one parent
		if len(f.Parents) != 1 {
			return fmt.Errorf("File %s does not have exacly one parent", f.Id)
		}

		// Ensure that there are no duplicate files
		uniqKey := f.Name + f.Parents[0]
		if dupeId, isDupe := uniq[uniqKey]; isDupe {
			return fmt.Errorf("Found name collision between %s and %s", f.Id, dupeId)
		}
		uniq[uniqKey] = f.Id
	}

	return nil
}

type LocalFile struct {
	absPath string
	relPath string
	info    os.FileInfo
}

type RemoteFile struct {
	relPath string
	file    *drive.File
}

type changedFile struct {
	local  *LocalFile
	remote *RemoteFile
}

type syncFiles struct {
	root    *RemoteFile
	local   []*LocalFile
	remote  []*RemoteFile
	compare FileComparer
}

type FileComparer interface {
	Changed(*LocalFile, *RemoteFile) bool
}

func (self LocalFile) AbsPath() string {
	return self.absPath
}

func (self LocalFile) Size() int64 {
	return self.info.Size()
}

func (self LocalFile) Modified() time.Time {
	return self.info.ModTime()
}

func (self RemoteFile) Md5() string {
	return self.file.Md5Checksum
}

func (self RemoteFile) Size() int64 {
	return self.file.Size
}

func (self RemoteFile) Modified() time.Time {
	t, _ := time.Parse(time.RFC3339, self.file.ModifiedTime)
	return t
}

func (self *changedFile) compareModTime() ModTime {
	localTime := self.local.Modified()
	remoteTime := self.remote.Modified()

	if localTime.After(remoteTime) {
		return LocalLastModified
	}

	if remoteTime.After(localTime) {
		return RemoteLastModified
	}

	return EqualModifiedTime
}

func (self *changedFile) compareSize() LargestSize {
	localSize := self.local.Size()
	remoteSize := self.remote.Size()

	if localSize > remoteSize {
		return LocalLargestSize
	}

	if remoteSize > localSize {
		return RemoteLargestSize
	}

	return EqualSize
}

func (self *syncFiles) filterMissingRemoteDirs() []*LocalFile {
	var files []*LocalFile

	for _, lf := range self.local {
		if lf.info.IsDir() && !self.existsRemote(lf) {
			files = append(files, lf)
		}
	}

	return files
}

func (self *syncFiles) filterMissingLocalDirs() []*RemoteFile {
	var files []*RemoteFile

	for _, rf := range self.remote {
		if isDir(rf.file) && !self.existsLocal(rf) {
			files = append(files, rf)
		}
	}

	return files
}

func (self *syncFiles) filterMissingRemoteFiles() []*LocalFile {
	var files []*LocalFile

	for _, lf := range self.local {
		if !lf.info.IsDir() && !self.existsRemote(lf) {
			files = append(files, lf)
		}
	}

	return files
}

func (self *syncFiles) filterMissingLocalFiles() []*RemoteFile {
	var files []*RemoteFile

	for _, rf := range self.remote {
		if !isDir(rf.file) && !self.existsLocal(rf) {
			files = append(files, rf)
		}
	}

	return files
}

func (self *syncFiles) filterChangedLocalFiles() []*changedFile {
	var files []*changedFile

	for _, lf := range self.local {
		// Skip directories
		if lf.info.IsDir() {
			continue
		}

		// Skip files that don't exist on drive
		rf, found := self.findRemoteByPath(lf.relPath)
		if !found {
			continue
		}

		// Check if file has changed
		if self.compare.Changed(lf, rf) {
			files = append(files, &changedFile{
				local:  lf,
				remote: rf,
			})
		}
	}

	return files
}

func (self *syncFiles) filterChangedRemoteFiles() []*changedFile {
	var files []*changedFile

	for _, rf := range self.remote {
		// Skip directories
		if isDir(rf.file) {
			continue
		}

		// Skip local files that don't exist
		lf, found := self.findLocalByPath(rf.relPath)
		if !found {
			continue
		}

		// Check if file has changed
		if self.compare.Changed(lf, rf) {
			files = append(files, &changedFile{
				local:  lf,
				remote: rf,
			})
		}
	}

	return files
}

func (self *syncFiles) filterExtraneousRemoteFiles() []*RemoteFile {
	var files []*RemoteFile

	for _, rf := range self.remote {
		if !self.existsLocal(rf) {
			files = append(files, rf)
		}
	}

	return files
}

func (self *syncFiles) filterExtraneousLocalFiles() []*LocalFile {
	var files []*LocalFile

	for _, lf := range self.local {
		if !self.existsRemote(lf) {
			files = append(files, lf)
		}
	}

	return files
}

func (self *syncFiles) existsRemote(lf *LocalFile) bool {
	_, found := self.findRemoteByPath(lf.relPath)
	return found
}

func (self *syncFiles) existsLocal(rf *RemoteFile) bool {
	_, found := self.findLocalByPath(rf.relPath)
	return found
}

func (self *syncFiles) findRemoteByPath(relPath string) (*RemoteFile, bool) {
	if relPath == "." {
		return self.root, true
	}

	for _, rf := range self.remote {
		if relPath == rf.relPath {
			return rf, true
		}
	}

	return nil, false
}

func (self *syncFiles) findLocalByPath(relPath string) (*LocalFile, bool) {
	for _, lf := range self.local {
		if relPath == lf.relPath {
			return lf, true
		}
	}

	return nil, false
}

func findLocalConflicts(files []*changedFile) []*changedFile {
	var conflicts []*changedFile

	for _, cf := range files {
		if cf.compareModTime() == LocalLastModified {
			conflicts = append(conflicts, cf)
		}
	}

	return conflicts
}

func findRemoteConflicts(files []*changedFile) []*changedFile {
	var conflicts []*changedFile

	for _, cf := range files {
		if cf.compareModTime() == RemoteLastModified {
			conflicts = append(conflicts, cf)
		}
	}

	return conflicts
}

type byLocalPathLength []*LocalFile

func (self byLocalPathLength) Len() int {
	return len(self)
}

func (self byLocalPathLength) Swap(i, j int) {
	self[i], self[j] = self[j], self[i]
}

func (self byLocalPathLength) Less(i, j int) bool {
	return pathLength(self[i].relPath) < pathLength(self[j].relPath)
}

type byRemotePathLength []*RemoteFile

func (self byRemotePathLength) Len() int {
	return len(self)
}

func (self byRemotePathLength) Swap(i, j int) {
	self[i], self[j] = self[j], self[i]
}

func (self byRemotePathLength) Less(i, j int) bool {
	return pathLength(self[i].relPath) < pathLength(self[j].relPath)
}

type byRemotePath []*RemoteFile

func (self byRemotePath) Len() int {
	return len(self)
}

func (self byRemotePath) Swap(i, j int) {
	self[i], self[j] = self[j], self[i]
}

func (self byRemotePath) Less(i, j int) bool {
	return strings.ToLower(self[i].relPath) < strings.ToLower(self[j].relPath)
}

type ignoreFunc func(string) bool

func prepareIgnorer(path string) (ignoreFunc, error) {
	acceptAll := func(string) bool {
		return false
	}

	if !fileExists(path) {
		return acceptAll, nil
	}

	ignorer, err := ignore.CompileIgnoreFile(path)
	if err != nil {
		return acceptAll, fmt.Errorf("Failed to prepare ignorer: %s", err)
	}

	return ignorer.MatchesPath, nil
}

func formatConflicts(conflicts []*changedFile, out io.Writer) {
	w := new(tabwriter.Writer)
	w.Init(out, 0, 0, 3, ' ', 0)

	fmt.Fprintln(w, "Path\tSize Local\tSize Remote\tModified Local\tModified Remote")

	for _, cf := range conflicts {
		fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\n",
			truncateString(cf.local.relPath, 60),
			formatSize(cf.local.Size(), false),
			formatSize(cf.remote.Size(), false),
			cf.local.Modified().Local().Format("Jan _2 2006 15:04:05.000"),
			cf.remote.Modified().Local().Format("Jan _2 2006 15:04:05.000"),
		)
	}

	w.Flush()
}