From 93f80f93413e30604788112a85c5d6f928d85d46 Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Mon, 10 Sep 2018 15:44:21 +0200 Subject: [PATCH 01/10] Start upstream earlier --- cmd/status_sync.go | 7 -- cmd/up.go | 12 +-- pkg/devspace/sync/downstream.go | 4 +- pkg/devspace/sync/file_information.go | 18 ++++ pkg/devspace/sync/sync_config.go | 138 +++++++++++++++++--------- pkg/devspace/sync/sync_config_test.go | 6 +- pkg/devspace/sync/upstream.go | 38 ++++--- 7 files changed, 141 insertions(+), 82 deletions(-) diff --git a/cmd/status_sync.go b/cmd/status_sync.go index 69c920ba33..9f3b32141c 100644 --- a/cmd/status_sync.go +++ b/cmd/status_sync.go @@ -15,7 +15,6 @@ import ( "github.com/spf13/cobra" ) -var initialSyncCompleted = regexp.MustCompile(`^\[Sync\] Initial sync completed\. Processed (\d+) changes$`) var syncStopped = regexp.MustCompile(`^\[Sync\] Sync stopped$`) var downstreamChanges = regexp.MustCompile(`^\[Downstream\] Successfully processed (\d+) change\(s\)$`) var upstreamChanges = regexp.MustCompile(`^\[Upstream\] Successfully processed (\d+) change\(s\)$`) @@ -180,12 +179,6 @@ func updateSyncMap(syncMap map[string]*syncStatus, jsonMap map[string]string) er syncMap[identifier].Status = "Error" syncMap[identifier].Error = message syncMap[identifier].LastActivityTime = time - } else if matches := initialSyncCompleted.FindStringSubmatch(message); len(matches) == 2 { - syncMap[identifier].LastActivity = "Initially transferred " + matches[1] + " changes" - syncMap[identifier].LastActivityTime = time - - changes, _ := strconv.Atoi(matches[1]) - syncMap[identifier].TotalChanges += changes } else if matches := downstreamChanges.FindStringSubmatch(message); len(matches) == 2 { syncMap[identifier].LastActivity = "Downloaded " + matches[1] + " changes" syncMap[identifier].LastActivityTime = time diff --git a/cmd/up.go b/cmd/up.go index cc4e644396..08c655903d 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -156,16 +156,12 @@ func (cmd *UpCmd) Run(cobraCmd *cobra.Command, args []string) { } } - if cmd.flags.deploy || shouldRebuild { + // Check if we find a running release pod + pod, err := getRunningDevSpacePod(cmd.helm, cmd.kubectl) + + if err != nil || cmd.flags.deploy || shouldRebuild { cmd.deployChart() } else { - // Check if we find a running release pod - pod, err := getRunningDevSpacePod(cmd.helm, cmd.kubectl) - - if err != nil { - log.Fatalf("Couldn't find running devspace pod: %s", err.Error()) - } - cmd.pod = pod } diff --git a/pkg/devspace/sync/downstream.go b/pkg/devspace/sync/downstream.go index 2c62f51ce4..98b2f6e99a 100644 --- a/pkg/devspace/sync/downstream.go +++ b/pkg/devspace/sync/downstream.go @@ -87,7 +87,9 @@ func (d *downstream) populateFileMap() error { defer d.config.fileIndex.fileMapMutex.Unlock() for _, element := range createFiles { - d.config.fileIndex.fileMap[element.Name] = element + if d.config.fileIndex.fileMap[element.Name] == nil { + d.config.fileIndex.fileMap[element.Name] = element + } } return nil diff --git a/pkg/devspace/sync/file_information.go b/pkg/devspace/sync/file_information.go index 8f43c4009d..0b274293de 100644 --- a/pkg/devspace/sync/file_information.go +++ b/pkg/devspace/sync/file_information.go @@ -4,6 +4,8 @@ import ( "strconv" "strings" + "github.com/rjeczalik/notify" + "github.com/juju/errors" ) @@ -29,6 +31,22 @@ type fileInformation struct { RemoteGID int // %u } +func (f *fileInformation) Sys() interface{} { + return f +} + +func (f *fileInformation) Path() string { + return f.Name +} + +func (f *fileInformation) Event() notify.Event { + if f.Mtime == 0 { + return notify.Remove + } + + return notify.Create +} + type parsingError struct { msg string } diff --git a/pkg/devspace/sync/sync_config.go b/pkg/devspace/sync/sync_config.go index 248810fcce..bd9c101d10 100644 --- a/pkg/devspace/sync/sync_config.go +++ b/pkg/devspace/sync/sync_config.go @@ -4,6 +4,8 @@ import ( "io/ioutil" "os" "path" + "sync" + "time" "github.com/covexo/devspace/pkg/util/log" "github.com/juju/errors" @@ -14,6 +16,7 @@ import ( "k8s.io/client-go/kubernetes" ) +var initialUpstreamBatchSize = 1000 var syncLog log.Logger //StartAck signals to the user that the sync process is starting @@ -47,7 +50,8 @@ type SyncConfig struct { upstream *upstream downstream *downstream - silent bool + silent bool + stopOnce sync.Once // Used for testing testing bool @@ -189,41 +193,48 @@ func (s *SyncConfig) initIgnoreParsers() error { func (s *SyncConfig) mainLoop() { s.Logf("[Sync] Start syncing") - err := s.initialSync() - if err != nil { - s.Error(err) - return - } + // Start upstream as early as possible + go s.startUpstream() - // Start upstream + // Start downstream and do initial sync go func() { defer s.Stop() - // Set up a watchpoint listening for events within a directory tree rooted at specified directory - err := notify.Watch(s.WatchPath+"/...", s.upstream.events, notify.All) - + err := s.initialSync() if err != nil { s.Error(err) return } - defer notify.Stop(s.upstream.events) - - err = s.upstream.mainLoop() - if err != nil { - s.Error(err) - } + s.startDownstream() }() +} - // Start downstream - go func() { - defer s.Stop() +func (s *SyncConfig) startUpstream() { + defer s.Stop() - err := s.downstream.mainLoop() - if err != nil { - s.Error(err) - } - }() + // Set up a watchpoint listening for events within a directory tree rooted at specified directory + err := notify.Watch(s.WatchPath+"/...", s.upstream.events, notify.All) + if err != nil { + s.Error(err) + return + } + + defer notify.Stop(s.upstream.events) + + err = s.upstream.mainLoop() + if err != nil { + s.Error(err) + } +} + +func (s *SyncConfig) startDownstream() { + defer s.Stop() + + err := s.downstream.mainLoop() + if err != nil { + s.Error(err) + } } func (s *SyncConfig) initialSync() error { @@ -232,9 +243,10 @@ func (s *SyncConfig) initialSync() error { return errors.Trace(err) } - remoteChanges := make([]*fileInformation, 0, 10) + localChanges := make([]*fileInformation, 0, 10) fileMapClone := make(map[string]*fileInformation) + s.fileIndex.fileMapMutex.Lock() for key, element := range s.fileIndex.fileMap { if element.IsSymbolicLink { continue @@ -242,32 +254,29 @@ func (s *SyncConfig) initialSync() error { fileMapClone[key] = element } + s.fileIndex.fileMapMutex.Unlock() - err = s.diffServerClient(s.WatchPath, &remoteChanges, fileMapClone) + err = s.diffServerClient(s.WatchPath, &localChanges, fileMapClone) if err != nil { return errors.Trace(err) } - if len(remoteChanges) > 0 { - err = s.upstream.applyCreates(remoteChanges) - if err != nil { - return errors.Trace(err) - } + if len(localChanges) > 0 { + go s.sendChangesToUpstream(localChanges) } if len(fileMapClone) > 0 { - localChanges := make([]*fileInformation, 0, len(fileMapClone)) + remoteChanges := make([]*fileInformation, 0, len(fileMapClone)) for _, element := range fileMapClone { - localChanges = append(localChanges, element) + remoteChanges = append(localChanges, element) } - err = s.downstream.applyChanges(localChanges, nil) + err = s.downstream.applyChanges(remoteChanges, nil) if err != nil { return errors.Trace(err) } } - s.Logf("[Sync] Initial sync completed. Processed %d changes", len(remoteChanges)+len(fileMapClone)) return nil } @@ -301,8 +310,14 @@ func (s *SyncConfig) diffServerClient(filepath string, sendChanges *[]*fileInfor } } + // Access fileMap safely + s.fileIndex.fileMapMutex.Lock() + isSymbolicLink := s.fileIndex.fileMap[relativePath] != nil && s.fileIndex.fileMap[relativePath].IsSymbolicLink + isLocalFileNewer := s.fileIndex.fileMap[relativePath] == nil || ceilMtime(stat.ModTime()) > s.fileIndex.fileMap[relativePath].Mtime+1 + s.fileIndex.fileMapMutex.Unlock() + // Exclude remote symlinks - if s.fileIndex.fileMap[relativePath] != nil && s.fileIndex.fileMap[relativePath].IsSymbolicLink { + if isSymbolicLink { return nil } @@ -311,7 +326,7 @@ func (s *SyncConfig) diffServerClient(filepath string, sendChanges *[]*fileInfor } // TODO: Handle the case when local files are older than in the container - if s.fileIndex.fileMap[relativePath] == nil || ceilMtime(stat.ModTime()) > s.fileIndex.fileMap[relativePath].Mtime+1 { + if isLocalFileNewer { *sendChanges = append(*sendChanges, &fileInformation{ Name: relativePath, Mtime: ceilMtime(stat.ModTime()), @@ -323,6 +338,33 @@ func (s *SyncConfig) diffServerClient(filepath string, sendChanges *[]*fileInfor return nil } +func (s *SyncConfig) sendChangesToUpstream(changes []*fileInformation) { + for j := 0; j < len(changes); j += initialUpstreamBatchSize { + // Wait till upstream channel is empty + for len(s.upstream.events) > 0 { + time.Sleep(time.Second) + } + + // Now we send them to upstream + sendBatch := make([]*fileInformation, 0, initialUpstreamBatchSize) + s.fileIndex.fileMapMutex.Lock() + + for i := j; i < (j+initialUpstreamBatchSize) && i < len(changes); i++ { + if s.fileIndex.fileMap[changes[i].Name] == nil || (s.fileIndex.fileMap[changes[i].Name] != nil && changes[i].Mtime > s.fileIndex.fileMap[changes[i].Name].Mtime) { + sendBatch = append(sendBatch, changes[i]) + } + } + + s.fileIndex.fileMapMutex.Unlock() + + // We do this out of the fileIndex lock, because otherwise this could cause a deadlock + // (Upstream waits in getfileInformationFromEvent and upstream.events buffer is full) + for i := 0; i < len(sendBatch); i++ { + s.upstream.events <- sendBatch[i] + } + } +} + func (s *SyncConfig) diffDir(filepath string, sendChanges *[]*fileInformation, downloadChanges map[string]*fileInformation) error { relativePath := getRelativeFromFullPath(filepath, s.WatchPath) files, err := ioutil.ReadDir(filepath) @@ -333,12 +375,16 @@ func (s *SyncConfig) diffDir(filepath string, sendChanges *[]*fileInformation, d } if len(files) == 0 { + s.fileIndex.fileMapMutex.Lock() + if s.fileIndex.fileMap[relativePath] == nil { *sendChanges = append(*sendChanges, &fileInformation{ Name: relativePath, IsDirectory: true, }) } + + s.fileIndex.fileMapMutex.Unlock() } for _, f := range files { @@ -350,12 +396,10 @@ func (s *SyncConfig) diffDir(filepath string, sendChanges *[]*fileInformation, d return nil } -//Stop stops the sync process +// Stop stops the sync process func (s *SyncConfig) Stop() { - if s.upstream != nil && s.upstream.interrupt != nil { - select { - case <-s.upstream.interrupt: - default: + s.stopOnce.Do(func() { + if s.upstream != nil && s.upstream.interrupt != nil { close(s.upstream.interrupt) if s.upstream.stdinPipe != nil { @@ -371,12 +415,8 @@ func (s *SyncConfig) Stop() { s.upstream.stderrPipe.Close() } } - } - if s.downstream != nil && s.downstream.interrupt != nil { - select { - case <-s.downstream.interrupt: - default: + if s.downstream != nil && s.downstream.interrupt != nil { close(s.downstream.interrupt) if s.downstream.stdinPipe != nil { @@ -392,7 +432,7 @@ func (s *SyncConfig) Stop() { s.downstream.stderrPipe.Close() } } - } - s.Logln("[Sync] Sync stopped") + s.Logln("[Sync] Sync stopped") + }) } diff --git a/pkg/devspace/sync/sync_config_test.go b/pkg/devspace/sync/sync_config_test.go index efedcf4313..44b71a245e 100644 --- a/pkg/devspace/sync/sync_config_test.go +++ b/pkg/devspace/sync/sync_config_test.go @@ -183,6 +183,9 @@ func TestInitialSync(t *testing.T) { return } + // TODO: Remove sleep and instead wait for upstream changes + time.Sleep(5 * time.Second) + // Check outcome filesToCheck := []string{ "testFile1", @@ -310,7 +313,8 @@ func TestRunningSync(t *testing.T) { } // Start sync and do initial sync - syncClient.mainLoop() + go syncClient.startUpstream() + go syncClient.startDownstream() // Create err = createFileAndWait(remote, local, "2") diff --git a/pkg/devspace/sync/upstream.go b/pkg/devspace/sync/upstream.go index 25feee23f6..fa2fd04a00 100644 --- a/pkg/devspace/sync/upstream.go +++ b/pkg/devspace/sync/upstream.go @@ -141,28 +141,34 @@ func (u *upstream) getfileInformationFromEvent(events []notify.EventInfo) []*fil changes := make([]*fileInformation, 0, len(events)) for _, event := range events { - fullpath := event.Path() - relativePath := getRelativeFromFullPath(fullpath, u.config.WatchPath) + fileInfo, ok := event.(*fileInformation) - // Exclude changes on the exclude list - if u.config.ignoreMatcher != nil { - if u.config.ignoreMatcher.MatchesPath(relativePath) { - continue + if ok { + changes = append(changes, fileInfo) + } else { + fullpath := event.Path() + relativePath := getRelativeFromFullPath(fullpath, u.config.WatchPath) + + // Exclude changes on the exclude list + if u.config.ignoreMatcher != nil { + if u.config.ignoreMatcher.MatchesPath(relativePath) { + continue + } } - } - // Exclude changes on the upload exclude list - if u.config.uploadIgnoreMatcher != nil { - if u.config.uploadIgnoreMatcher.MatchesPath(relativePath) { - continue + // Exclude changes on the upload exclude list + if u.config.uploadIgnoreMatcher != nil { + if u.config.uploadIgnoreMatcher.MatchesPath(relativePath) { + continue + } } - } - // Determine what kind of change we got (Create or Remove) - newChange := evaluateChange(fileMap, relativePath, fullpath) + // Determine what kind of change we got (Create or Remove) + newChange := evaluateChange(fileMap, relativePath, fullpath) - if newChange != nil { - changes = append(changes, newChange) + if newChange != nil { + changes = append(changes, newChange) + } } } From 8ad8115b722fbc5bd5a54f976fdb090652346571 Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Mon, 10 Sep 2018 17:33:28 +0200 Subject: [PATCH 02/10] Move logic when to sync files to separate file --- pkg/devspace/sync/downstream.go | 88 +++----------- pkg/devspace/sync/evaluater.go | 199 +++++++++++++++++++++++++++++++ pkg/devspace/sync/sync_config.go | 91 +++++--------- pkg/devspace/sync/tar.go | 2 +- pkg/devspace/sync/upstream.go | 67 +++-------- pkg/devspace/sync/util.go | 51 +++----- 6 files changed, 277 insertions(+), 221 deletions(-) create mode 100644 pkg/devspace/sync/evaluater.go diff --git a/pkg/devspace/sync/downstream.go b/pkg/devspace/sync/downstream.go index 98b2f6e99a..b4c2692bae 100644 --- a/pkg/devspace/sync/downstream.go +++ b/pkg/devspace/sync/downstream.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "strconv" "strings" "time" @@ -330,36 +331,28 @@ func (d *downstream) removeFilesAndFolders(removeFiles map[string]*fileInformati d.config.Logf("[Downstream] Remove %d files", numRemoveFiles) } - // A file is only deleted if the following conditions are met: - // - The file name is present in the d.config.fileMap map - // - The file did not change in terms of size and mtime in the d.config.fileMap since we started the collecting changes process - // - The file is present on the filesystem and did not change in terms of size and mtime on the filesystem for key, value := range removeFiles { - if value != nil && fileMap[key] != nil { - // Exclude files on the exclude list - if d.config.downloadIgnoreMatcher != nil { - if d.config.downloadIgnoreMatcher.MatchesPath(key) { - delete(fileMap, key) - continue - } - } + absFilepath := filepath.Join(d.config.WatchPath, key) + if shouldRemoveLocal(absFilepath, value, d.config) { if numRemoveFiles <= 3 { d.config.Logf("[Downstream] Remove %s", key) } - if fileMap[key].IsDirectory { + if value.IsDirectory { deleteSafeRecursive(d.config.WatchPath, key, fileMap, removeFiles, d.config) } else { - if value.Mtime == fileMap[key].Mtime && value.Size == fileMap[key].Size { - if deleteSafe(path.Join(d.config.WatchPath, key), fileMap[key]) == false { - d.config.Logf("[Downstream] Skip file delete %s", key) - } + err := os.Remove(absFilepath) + if err != nil { + d.config.Logf("[Downstream] Skip file delete %s: %v", key, err) + continue } - - delete(fileMap, key) } + } else { + d.config.Logf("[Downstream] Skip delete %s", key) } + + delete(fileMap, key) } } @@ -483,7 +476,6 @@ func (d *downstream) evaluateFile(fileline string, createFiles *[]*fileInformati d.config.fileIndex.fileMapMutex.Lock() defer d.config.fileIndex.fileMapMutex.Unlock() - fileMap := d.config.fileIndex.fileMap fileInformation, err := parseFileInformation(fileline, d.config.DestPath) // Error parsing line @@ -496,59 +488,11 @@ func (d *downstream) evaluateFile(fileline string, createFiles *[]*fileInformati return nil } - // Exclude files on the exclude list - if d.config.ignoreMatcher != nil { - if d.config.ignoreMatcher.MatchesPath(fileInformation.Name) { - return nil - } - } - - // File found, don't delete it - if removeFiles[fileInformation.Name] != nil { - delete(removeFiles, fileInformation.Name) - } - - // Update mode, gid & uid if exists - if fileMap[fileInformation.Name] != nil { - fileMap[fileInformation.Name].RemoteMode = fileInformation.RemoteMode - fileMap[fileInformation.Name].RemoteGID = fileInformation.RemoteGID - fileMap[fileInformation.Name].RemoteUID = fileInformation.RemoteUID - } - - // Exclude files on the exclude list - if d.config.downloadIgnoreMatcher != nil { - if d.config.downloadIgnoreMatcher.MatchesPath(fileInformation.Name) { - return nil - } - } - - // Exclude symlinks - if fileInformation.IsSymbolicLink { - // Add them to the fileMap though - fileMap[fileInformation.Name] = fileInformation - return nil - } - - // Does file already exist in the filemap? - if fileMap[fileInformation.Name] != nil { - // Don't override folders that exist in the filemap - if fileInformation.IsDirectory == false { - // Redownload file if mtime is newer than saved one - if fileInformation.Mtime > fileMap[fileInformation.Name].Mtime { - *createFiles = append(*createFiles, fileInformation) - - return nil - } + // File found don't delete it + delete(removeFiles, fileInformation.Name) - // Redownload file if size changed && file is not older than the one in the fileMap - // the mTime check is necessary, because otherwise we would override older local files that - // are not overridden initially - if fileInformation.Mtime == fileMap[fileInformation.Name].Mtime && fileInformation.Size != fileMap[fileInformation.Name].Size { - *createFiles = append(*createFiles, fileInformation) - } - } - } else { - // We create the file if it doesn't exist in the fileMap + // Should we download the file / folder? + if shouldDownload(fileInformation, d.config) { *createFiles = append(*createFiles, fileInformation) } diff --git a/pkg/devspace/sync/evaluater.go b/pkg/devspace/sync/evaluater.go new file mode 100644 index 0000000000..c042b399e2 --- /dev/null +++ b/pkg/devspace/sync/evaluater.go @@ -0,0 +1,199 @@ +package sync + +import ( + "os" +) + +// s.fileIndex needs to be locked before this function is called +func shouldRemoveRemote(relativePath string, s *SyncConfig) bool { + // Exclude changes on the exclude list + if s.ignoreMatcher != nil { + if s.ignoreMatcher.MatchesPath(relativePath) { + return false + } + } + + // Exclude changes on the upload exclude list + if s.uploadIgnoreMatcher != nil { + if s.uploadIgnoreMatcher.MatchesPath(relativePath) { + return false + } + } + + // File / Folder was already deleted from map so event was already processed or should not be processed + if s.fileIndex.fileMap[relativePath] == nil { + return false + } + + // Exclude symbolic links + if s.fileIndex.fileMap[relativePath].IsSymbolicLink { + return false + } + + return true +} + +// s.fileIndex needs to be locked before this function is called +func shouldUpload(relativePath string, stat os.FileInfo, s *SyncConfig, isInitial bool) bool { + // Exclude if stat is nil + if stat == nil { + return false + } + + // Exclude changes on the exclude list + if s.ignoreMatcher != nil { + if s.ignoreMatcher.MatchesPath(relativePath) { + return false + } + } + + // Exclude changes on the upload exclude list + if s.uploadIgnoreMatcher != nil { + if s.uploadIgnoreMatcher.MatchesPath(relativePath) { + // Add to file map and prevent download if local file is newer than the remote one + if s.fileIndex.fileMap[relativePath] != nil && s.fileIndex.fileMap[relativePath].Mtime < ceilMtime(stat.ModTime()) { + fileInformation := &fileInformation{ + Name: relativePath, + Mtime: ceilMtime(stat.ModTime()), + Size: stat.Size(), + IsDirectory: stat.IsDir(), + } + + // Add it to the fileMap + s.fileIndex.fileMap[relativePath] = fileInformation + } + + return false + } + } + + // Exclude local symlinks + if stat.Mode()&os.ModeSymlink != 0 { + return false + } + + // Check if we already tracked the path + if s.fileIndex.fileMap[relativePath] != nil { + // Folder already exists + if stat.IsDir() { + return false + } + + // Exclude symlinks + if s.fileIndex.fileMap[relativePath].IsSymbolicLink { + return false + } + + if isInitial { + // File is older locally than remote so don't update remote + if ceilMtime(stat.ModTime()) <= s.fileIndex.fileMap[relativePath].Mtime+1 { + return false + } + } else { + // File did not change or was changed by downstream + if ceilMtime(stat.ModTime()) == s.fileIndex.fileMap[relativePath].Mtime && stat.Size() == s.fileIndex.fileMap[relativePath].Size { + return false + } + } + } + + return true +} + +// s.fileIndex needs to be locked before this function is called +func shouldDownload(fileInformation *fileInformation, s *SyncConfig) bool { + // Exclude files on the exclude list + if s.ignoreMatcher != nil { + if s.ignoreMatcher.MatchesPath(fileInformation.Name) { + return false + } + } + + // Update mode, gid & uid if exists + if s.fileIndex.fileMap[fileInformation.Name] != nil { + s.fileIndex.fileMap[fileInformation.Name].RemoteMode = fileInformation.RemoteMode + s.fileIndex.fileMap[fileInformation.Name].RemoteGID = fileInformation.RemoteGID + s.fileIndex.fileMap[fileInformation.Name].RemoteUID = fileInformation.RemoteUID + } + + // Exclude files on the exclude list + if s.downloadIgnoreMatcher != nil { + if s.downloadIgnoreMatcher.MatchesPath(fileInformation.Name) { + return false + } + } + + // Exclude symlinks + if fileInformation.IsSymbolicLink { + // Add them to the fileMap though + s.fileIndex.fileMap[fileInformation.Name] = fileInformation + return false + } + + // Does file already exist in the filemap? + if s.fileIndex.fileMap[fileInformation.Name] != nil { + // Don't override folders that exist in the filemap + if fileInformation.IsDirectory == false { + // Redownload file if mtime is newer than saved one + if fileInformation.Mtime > s.fileIndex.fileMap[fileInformation.Name].Mtime { + return true + } + + // Redownload file if size changed && file is not older than the one in the fileMap + // the mTime check is necessary, because otherwise we would override older local files that + // are not overridden initially + if fileInformation.Mtime == s.fileIndex.fileMap[fileInformation.Name].Mtime && fileInformation.Size != s.fileIndex.fileMap[fileInformation.Name].Size { + return true + } + } + + return false + } + + return true +} + +// s.fileIndex needs to be locked before this function is called +// A file is only deleted if the following conditions are met: +// - The file name is present in the d.config.fileMap map +// - The file did not change in terms of size and mtime in the d.config.fileMap since we started the collecting changes process +// - The file is present on the filesystem and did not change in terms of size and mtime on the filesystem +func shouldRemoveLocal(absFilepath string, fileInformation *fileInformation, s *SyncConfig) bool { + if fileInformation == nil { + return false + } + + // Exclude files on the exclude list + if s.downloadIgnoreMatcher != nil { + if s.downloadIgnoreMatcher.MatchesPath(fileInformation.Name) { + return false + } + } + + // Only delete if mtime and size did not change + stat, err := os.Stat(absFilepath) + if err != nil { + return false + } + + // We don't delete the file if we haven't tracked it + if stat != nil && s.fileIndex.fileMap[fileInformation.Name] != nil { + if stat.IsDir() != s.fileIndex.fileMap[fileInformation.Name].IsDirectory || stat.IsDir() != fileInformation.IsDirectory { + return false + } + + if fileInformation.IsDirectory == false { + // We don't delete the file if it has changed in the map since we collected changes + if fileInformation.Mtime == s.fileIndex.fileMap[fileInformation.Name].Mtime && fileInformation.Size == s.fileIndex.fileMap[fileInformation.Name].Size { + // We don't delete the file if it has changed on the filesystem meanwhile + if ceilMtime(stat.ModTime()) <= fileInformation.Mtime { + return true + } + } + } else { + return true + } + } + + return false +} diff --git a/pkg/devspace/sync/sync_config.go b/pkg/devspace/sync/sync_config.go index bd9c101d10..8209e7ccfe 100644 --- a/pkg/devspace/sync/sync_config.go +++ b/pkg/devspace/sync/sync_config.go @@ -206,6 +206,7 @@ func (s *SyncConfig) mainLoop() { return } + s.Logf("[Sync] Initial sync completed") s.startDownstream() }() } @@ -289,35 +290,13 @@ func (s *SyncConfig) diffServerClient(filepath string, sendChanges *[]*fileInfor return nil } - // We skip symlinks - if stat.Mode()&os.ModeSymlink != 0 { - return nil - } - - // Exclude files on the exclude list - if s.ignoreMatcher != nil { - if s.ignoreMatcher.MatchesPath(relativePath) { - return nil - } - } - delete(downloadChanges, relativePath) - // Exclude files on the exclude list - if s.uploadIgnoreMatcher != nil { - if s.uploadIgnoreMatcher.MatchesPath(relativePath) { - return nil - } - } - - // Access fileMap safely s.fileIndex.fileMapMutex.Lock() - isSymbolicLink := s.fileIndex.fileMap[relativePath] != nil && s.fileIndex.fileMap[relativePath].IsSymbolicLink - isLocalFileNewer := s.fileIndex.fileMap[relativePath] == nil || ceilMtime(stat.ModTime()) > s.fileIndex.fileMap[relativePath].Mtime+1 + shouldUpload := shouldUpload(relativePath, stat, s, true) s.fileIndex.fileMapMutex.Unlock() - // Exclude remote symlinks - if isSymbolicLink { + if shouldUpload == false { return nil } @@ -325,16 +304,39 @@ func (s *SyncConfig) diffServerClient(filepath string, sendChanges *[]*fileInfor return s.diffDir(filepath, sendChanges, downloadChanges) } - // TODO: Handle the case when local files are older than in the container - if isLocalFileNewer { + // Add file to upload + *sendChanges = append(*sendChanges, &fileInformation{ + Name: relativePath, + Mtime: ceilMtime(stat.ModTime()), + Size: stat.Size(), + IsDirectory: false, + }) + + return nil +} + +func (s *SyncConfig) diffDir(filepath string, sendChanges *[]*fileInformation, downloadChanges map[string]*fileInformation) error { + relativePath := getRelativeFromFullPath(filepath, s.WatchPath) + files, err := ioutil.ReadDir(filepath) + + if err != nil { + s.Logf("[Upstream] Couldn't read dir %s: %v", filepath, err) + return nil + } + + if len(files) == 0 { *sendChanges = append(*sendChanges, &fileInformation{ Name: relativePath, - Mtime: ceilMtime(stat.ModTime()), - Size: stat.Size(), - IsDirectory: false, + IsDirectory: true, }) } + for _, f := range files { + if err := s.diffServerClient(path.Join(filepath, f.Name()), sendChanges, downloadChanges); err != nil { + return errors.Trace(err) + } + } + return nil } @@ -365,37 +367,6 @@ func (s *SyncConfig) sendChangesToUpstream(changes []*fileInformation) { } } -func (s *SyncConfig) diffDir(filepath string, sendChanges *[]*fileInformation, downloadChanges map[string]*fileInformation) error { - relativePath := getRelativeFromFullPath(filepath, s.WatchPath) - files, err := ioutil.ReadDir(filepath) - - if err != nil { - s.Logf("[Upstream] Couldn't read dir %s: %v", filepath, err) - return nil - } - - if len(files) == 0 { - s.fileIndex.fileMapMutex.Lock() - - if s.fileIndex.fileMap[relativePath] == nil { - *sendChanges = append(*sendChanges, &fileInformation{ - Name: relativePath, - IsDirectory: true, - }) - } - - s.fileIndex.fileMapMutex.Unlock() - } - - for _, f := range files { - if err := s.diffServerClient(path.Join(filepath, f.Name()), sendChanges, downloadChanges); err != nil { - return errors.Trace(err) - } - } - - return nil -} - // Stop stops the sync process func (s *SyncConfig) Stop() { s.stopOnce.Do(func() { diff --git a/pkg/devspace/sync/tar.go b/pkg/devspace/sync/tar.go index 60108a0e63..651a2adb78 100644 --- a/pkg/devspace/sync/tar.go +++ b/pkg/devspace/sync/tar.go @@ -71,7 +71,7 @@ func untarNext(tarReader *tar.Reader, destPath, prefix string, config *SyncConfi IsDirectory: stat.IsDir(), } - config.Logf("[Downstream] Don't override %s because file has newer mTime timestamp\n", relativePath) + config.Logf("[Downstream] Don't override %s because file has newer mTime timestamp", relativePath) return true, nil } } diff --git a/pkg/devspace/sync/upstream.go b/pkg/devspace/sync/upstream.go index fa2fd04a00..3c73a53cd3 100644 --- a/pkg/devspace/sync/upstream.go +++ b/pkg/devspace/sync/upstream.go @@ -149,22 +149,8 @@ func (u *upstream) getfileInformationFromEvent(events []notify.EventInfo) []*fil fullpath := event.Path() relativePath := getRelativeFromFullPath(fullpath, u.config.WatchPath) - // Exclude changes on the exclude list - if u.config.ignoreMatcher != nil { - if u.config.ignoreMatcher.MatchesPath(relativePath) { - continue - } - } - - // Exclude changes on the upload exclude list - if u.config.uploadIgnoreMatcher != nil { - if u.config.uploadIgnoreMatcher.MatchesPath(relativePath) { - continue - } - } - // Determine what kind of change we got (Create or Remove) - newChange := evaluateChange(fileMap, relativePath, fullpath) + newChange := evaluateChange(u.config, fileMap, relativePath, fullpath) if newChange != nil { changes = append(changes, newChange) @@ -175,52 +161,31 @@ func (u *upstream) getfileInformationFromEvent(events []notify.EventInfo) []*fil return changes } -func evaluateChange(fileMap map[string]*fileInformation, relativePath, fullpath string) *fileInformation { +func evaluateChange(s *SyncConfig, fileMap map[string]*fileInformation, relativePath, fullpath string) *fileInformation { stat, err := os.Stat(fullpath) // File / Folder exist -> Create File or Folder // if File / Folder does not exist, we create a new remove change if err == nil { - if fileMap[relativePath] != nil { - // Folder already exists - if stat.IsDir() { - return nil - } - - // File did not change or was changed by downstream - if ceilMtime(stat.ModTime()) == fileMap[relativePath].Mtime && stat.Size() == fileMap[relativePath].Size { - return nil - } - - // Exclude symbolic links - if fileMap[relativePath].IsSymbolicLink { - return nil + if shouldUpload(relativePath, stat, s, false) { + // New Create Task + return &fileInformation{ + Name: relativePath, + Mtime: ceilMtime(stat.ModTime()), + Size: stat.Size(), + IsDirectory: stat.IsDir(), } } - - // New Create Task - return &fileInformation{ - Name: relativePath, - Mtime: ceilMtime(stat.ModTime()), - Size: stat.Size(), - IsDirectory: stat.IsDir(), + } else { + if shouldRemoveRemote(relativePath, s) { + // New Remove Task + return &fileInformation{ + Name: relativePath, + } } } - // File / Folder was already deleted from map so event was already processed or should not be processed - if fileMap[relativePath] == nil { - return nil - } - - // Exclude symbolic links - if fileMap[relativePath].IsSymbolicLink { - return nil - } - - // New Remove Task - return &fileInformation{ - Name: relativePath, - } + return nil } func (u *upstream) applyChanges(changes []*fileInformation) error { diff --git a/pkg/devspace/sync/util.go b/pkg/devspace/sync/util.go index a564567f50..b0e94a0911 100644 --- a/pkg/devspace/sync/util.go +++ b/pkg/devspace/sync/util.go @@ -241,67 +241,44 @@ func deleteSafeRecursive(basepath, relativePath string, fileMap map[string]*file // We don't delete the folder or the contents if we haven't tracked it if fileMap[relativePath] == nil || removeFiles[relativePath] == nil { config.Logf("[Downstream] Skip delete directory %s\n", relativePath) - return } // Delete directory from fileMap defer delete(fileMap, relativePath) files, err := ioutil.ReadDir(absolutePath) - if err != nil { return } for _, f := range files { - if f.IsDir() { - deleteSafeRecursive(basepath, path.Join(relativePath, f.Name()), fileMap, removeFiles, config) - } else { - filepath := path.Join(relativePath, f.Name()) - fileDeleted := false - - // We don't delete the file if we haven't tracked it - if fileMap[filepath] != nil && removeFiles[filepath] != nil { - // We don't delete the file if it has changed in the map since we collected changes - if removeFiles[filepath].Mtime == fileMap[filepath].Mtime && removeFiles[filepath].Size == fileMap[filepath].Size { - // We don't delete the file if it has changed on the filesystem meanwhile - fileDeleted = deleteSafe(path.Join(basepath, filepath), fileMap[filepath]) - } - } + filepath := path.Join(relativePath, f.Name()) + absFilepath := path.Join(basepath, filepath) - if fileDeleted == false { - config.Logf("[Downstream] Skip file delete %s\n", relativePath) + if shouldRemoveLocal(absFilepath, fileMap[filepath], config) { + if f.IsDir() { + deleteSafeRecursive(basepath, filepath, fileMap, removeFiles, config) } else { - delete(fileMap, filepath) + err = os.Remove(absFilepath) + if err != nil { + config.Logf("[Downstream] Skip file delete %s: %v", relativePath, err) + continue + } } + } else { + config.Logf("[Downstream] Skip delete %s", relativePath) } + + delete(fileMap, filepath) } // This will not remove the directory if there is still a file or directory in it err = os.Remove(absolutePath) - if err != nil { config.Logf("[Downstream] Skip delete directory %s, because %s\n", relativePath, err.Error()) } } -func deleteSafe(path string, fileInformation *fileInformation) bool { - // Only delete if mtime and size did not change - stat, err := os.Stat(path) - - // TODO: uncomment this line for more safety (However we have to change the initial sync functionality that older files locally are either uplaoded or the newer files on the server downloaded) - // if err == nil && stat.Size() == fileInformation.Size && ceilMtime(stat.ModTime()) == fileInformation.Mtime { - if err == nil && ceilMtime(stat.ModTime()) <= fileInformation.Mtime { - err = os.Remove(path) - - if err == nil { - return true - } - } - - return false -} - func compilePaths(excludePaths []string) (gitignore.IgnoreParser, error) { if len(excludePaths) > 0 { ignoreParser, err := gitignore.CompileIgnoreLines(excludePaths...) From d1ff6756fc850ccfaeea58bb1dea5840db1e3c9e Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Mon, 10 Sep 2018 18:10:38 +0200 Subject: [PATCH 03/10] Fix test problem --- pkg/devspace/sync/sync_config_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/devspace/sync/sync_config_test.go b/pkg/devspace/sync/sync_config_test.go index 44b71a245e..49da220843 100644 --- a/pkg/devspace/sync/sync_config_test.go +++ b/pkg/devspace/sync/sync_config_test.go @@ -176,6 +176,8 @@ func TestInitialSync(t *testing.T) { ioutil.WriteFile(path.Join(remote, "testFolder", "testFile3"), []byte(fileContents), 0666) ioutil.WriteFile(path.Join(remote, "testFolder", "testFile4"), []byte(fileContents), 0666) + go syncClient.startUpstream() + // Do initial sync err = syncClient.initialSync() if err != nil { From 96d0c1e55e49075f1402db49592a8b47cad5c3dd Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Mon, 10 Sep 2018 18:18:02 +0200 Subject: [PATCH 04/10] Fix sync test problem --- pkg/devspace/sync/sync_config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/devspace/sync/sync_config.go b/pkg/devspace/sync/sync_config.go index 8209e7ccfe..32157730d1 100644 --- a/pkg/devspace/sync/sync_config.go +++ b/pkg/devspace/sync/sync_config.go @@ -269,7 +269,7 @@ func (s *SyncConfig) initialSync() error { if len(fileMapClone) > 0 { remoteChanges := make([]*fileInformation, 0, len(fileMapClone)) for _, element := range fileMapClone { - remoteChanges = append(localChanges, element) + remoteChanges = append(remoteChanges, element) } err = s.downstream.applyChanges(remoteChanges, nil) From 4938075f1e182cbedbdf6916f3d50d849bbdbf78 Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Mon, 10 Sep 2018 18:28:09 +0200 Subject: [PATCH 05/10] Fix test sync problem --- pkg/devspace/sync/sync_config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/devspace/sync/sync_config.go b/pkg/devspace/sync/sync_config.go index 32157730d1..8588ec7614 100644 --- a/pkg/devspace/sync/sync_config.go +++ b/pkg/devspace/sync/sync_config.go @@ -353,6 +353,7 @@ func (s *SyncConfig) sendChangesToUpstream(changes []*fileInformation) { for i := j; i < (j+initialUpstreamBatchSize) && i < len(changes); i++ { if s.fileIndex.fileMap[changes[i].Name] == nil || (s.fileIndex.fileMap[changes[i].Name] != nil && changes[i].Mtime > s.fileIndex.fileMap[changes[i].Name].Mtime) { + s.Logf("Send change %v to upstream", changes[i]) sendBatch = append(sendBatch, changes[i]) } } From 9603cde6fa583f7bd0ed0b172a3807ea1d588da2 Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Mon, 10 Sep 2018 18:32:14 +0200 Subject: [PATCH 06/10] Sync test --- pkg/devspace/sync/sync_config.go | 1 + 1 file changed, 1 insertion(+) diff --git a/pkg/devspace/sync/sync_config.go b/pkg/devspace/sync/sync_config.go index 8588ec7614..ad16aec559 100644 --- a/pkg/devspace/sync/sync_config.go +++ b/pkg/devspace/sync/sync_config.go @@ -352,6 +352,7 @@ func (s *SyncConfig) sendChangesToUpstream(changes []*fileInformation) { s.fileIndex.fileMapMutex.Lock() for i := j; i < (j+initialUpstreamBatchSize) && i < len(changes); i++ { + s.Logf("Change %v", changes[i]) if s.fileIndex.fileMap[changes[i].Name] == nil || (s.fileIndex.fileMap[changes[i].Name] != nil && changes[i].Mtime > s.fileIndex.fileMap[changes[i].Name].Mtime) { s.Logf("Send change %v to upstream", changes[i]) sendBatch = append(sendBatch, changes[i]) From bd7d930ea11eb58b93f77ab987b3c6929c3a3223 Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Mon, 10 Sep 2018 18:37:13 +0200 Subject: [PATCH 07/10] Fix initial sync problem --- pkg/devspace/sync/evaluater.go | 4 +++- pkg/devspace/sync/sync_config.go | 22 +++++++++------------- 2 files changed, 12 insertions(+), 14 deletions(-) diff --git a/pkg/devspace/sync/evaluater.go b/pkg/devspace/sync/evaluater.go index c042b399e2..0658a0a851 100644 --- a/pkg/devspace/sync/evaluater.go +++ b/pkg/devspace/sync/evaluater.go @@ -76,7 +76,9 @@ func shouldUpload(relativePath string, stat os.FileInfo, s *SyncConfig, isInitia if s.fileIndex.fileMap[relativePath] != nil { // Folder already exists if stat.IsDir() { - return false + // We want to initially walk over all files therefore we return true for a directory + // Later on a created directory locally that already exists in the fileMap should be ignored + return isInitial } // Exclude symlinks diff --git a/pkg/devspace/sync/sync_config.go b/pkg/devspace/sync/sync_config.go index ad16aec559..b6d7e10f98 100644 --- a/pkg/devspace/sync/sync_config.go +++ b/pkg/devspace/sync/sync_config.go @@ -296,21 +296,19 @@ func (s *SyncConfig) diffServerClient(filepath string, sendChanges *[]*fileInfor shouldUpload := shouldUpload(relativePath, stat, s, true) s.fileIndex.fileMapMutex.Unlock() - if shouldUpload == false { - return nil - } - if stat.IsDir() { return s.diffDir(filepath, sendChanges, downloadChanges) } - // Add file to upload - *sendChanges = append(*sendChanges, &fileInformation{ - Name: relativePath, - Mtime: ceilMtime(stat.ModTime()), - Size: stat.Size(), - IsDirectory: false, - }) + if shouldUpload == false { + // Add file to upload + *sendChanges = append(*sendChanges, &fileInformation{ + Name: relativePath, + Mtime: ceilMtime(stat.ModTime()), + Size: stat.Size(), + IsDirectory: false, + }) + } return nil } @@ -352,9 +350,7 @@ func (s *SyncConfig) sendChangesToUpstream(changes []*fileInformation) { s.fileIndex.fileMapMutex.Lock() for i := j; i < (j+initialUpstreamBatchSize) && i < len(changes); i++ { - s.Logf("Change %v", changes[i]) if s.fileIndex.fileMap[changes[i].Name] == nil || (s.fileIndex.fileMap[changes[i].Name] != nil && changes[i].Mtime > s.fileIndex.fileMap[changes[i].Name].Mtime) { - s.Logf("Send change %v to upstream", changes[i]) sendBatch = append(sendBatch, changes[i]) } } From 11bf47fff32e8613eea0f5b94fcb39c3e946145f Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Mon, 10 Sep 2018 19:08:56 +0200 Subject: [PATCH 08/10] Test sync --- pkg/devspace/sync/sync_config.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pkg/devspace/sync/sync_config.go b/pkg/devspace/sync/sync_config.go index b6d7e10f98..6cb21d9f1c 100644 --- a/pkg/devspace/sync/sync_config.go +++ b/pkg/devspace/sync/sync_config.go @@ -296,6 +296,8 @@ func (s *SyncConfig) diffServerClient(filepath string, sendChanges *[]*fileInfor shouldUpload := shouldUpload(relativePath, stat, s, true) s.fileIndex.fileMapMutex.Unlock() + s.Logf("shouldUpload %s %v", relativePath, relativePath) + if stat.IsDir() { return s.diffDir(filepath, sendChanges, downloadChanges) } @@ -350,8 +352,12 @@ func (s *SyncConfig) sendChangesToUpstream(changes []*fileInformation) { s.fileIndex.fileMapMutex.Lock() for i := j; i < (j+initialUpstreamBatchSize) && i < len(changes); i++ { + s.Logf("Change ", changes[i]) + if s.fileIndex.fileMap[changes[i].Name] == nil || (s.fileIndex.fileMap[changes[i].Name] != nil && changes[i].Mtime > s.fileIndex.fileMap[changes[i].Name].Mtime) { sendBatch = append(sendBatch, changes[i]) + + s.Logf("Send Change ", changes[i]) } } From 5a3648350ab574f7a16d0a64154e139662602784 Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Mon, 10 Sep 2018 19:17:56 +0200 Subject: [PATCH 09/10] Fix sync issue --- pkg/devspace/sync/sync_config.go | 24 ++++++++++-------------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/pkg/devspace/sync/sync_config.go b/pkg/devspace/sync/sync_config.go index 6cb21d9f1c..32157730d1 100644 --- a/pkg/devspace/sync/sync_config.go +++ b/pkg/devspace/sync/sync_config.go @@ -296,21 +296,21 @@ func (s *SyncConfig) diffServerClient(filepath string, sendChanges *[]*fileInfor shouldUpload := shouldUpload(relativePath, stat, s, true) s.fileIndex.fileMapMutex.Unlock() - s.Logf("shouldUpload %s %v", relativePath, relativePath) + if shouldUpload == false { + return nil + } if stat.IsDir() { return s.diffDir(filepath, sendChanges, downloadChanges) } - if shouldUpload == false { - // Add file to upload - *sendChanges = append(*sendChanges, &fileInformation{ - Name: relativePath, - Mtime: ceilMtime(stat.ModTime()), - Size: stat.Size(), - IsDirectory: false, - }) - } + // Add file to upload + *sendChanges = append(*sendChanges, &fileInformation{ + Name: relativePath, + Mtime: ceilMtime(stat.ModTime()), + Size: stat.Size(), + IsDirectory: false, + }) return nil } @@ -352,12 +352,8 @@ func (s *SyncConfig) sendChangesToUpstream(changes []*fileInformation) { s.fileIndex.fileMapMutex.Lock() for i := j; i < (j+initialUpstreamBatchSize) && i < len(changes); i++ { - s.Logf("Change ", changes[i]) - if s.fileIndex.fileMap[changes[i].Name] == nil || (s.fileIndex.fileMap[changes[i].Name] != nil && changes[i].Mtime > s.fileIndex.fileMap[changes[i].Name].Mtime) { sendBatch = append(sendBatch, changes[i]) - - s.Logf("Send Change ", changes[i]) } } From d2ad98c87576dada8d4ac16c34fbd3bbc0359646 Mon Sep 17 00:00:00 2001 From: Fabian Kramm Date: Mon, 10 Sep 2018 19:26:22 +0200 Subject: [PATCH 10/10] Fix initial sync upload dir problem --- pkg/devspace/sync/sync_config.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/devspace/sync/sync_config.go b/pkg/devspace/sync/sync_config.go index 32157730d1..4683c4e394 100644 --- a/pkg/devspace/sync/sync_config.go +++ b/pkg/devspace/sync/sync_config.go @@ -301,7 +301,7 @@ func (s *SyncConfig) diffServerClient(filepath string, sendChanges *[]*fileInfor } if stat.IsDir() { - return s.diffDir(filepath, sendChanges, downloadChanges) + return s.diffDir(filepath, stat, sendChanges, downloadChanges) } // Add file to upload @@ -315,7 +315,7 @@ func (s *SyncConfig) diffServerClient(filepath string, sendChanges *[]*fileInfor return nil } -func (s *SyncConfig) diffDir(filepath string, sendChanges *[]*fileInformation, downloadChanges map[string]*fileInformation) error { +func (s *SyncConfig) diffDir(filepath string, stat os.FileInfo, sendChanges *[]*fileInformation, downloadChanges map[string]*fileInformation) error { relativePath := getRelativeFromFullPath(filepath, s.WatchPath) files, err := ioutil.ReadDir(filepath) @@ -327,6 +327,8 @@ func (s *SyncConfig) diffDir(filepath string, sendChanges *[]*fileInformation, d if len(files) == 0 { *sendChanges = append(*sendChanges, &fileInformation{ Name: relativePath, + Mtime: ceilMtime(stat.ModTime()), + Size: stat.Size(), IsDirectory: true, }) }