diff --git a/cmd/status_sync.go b/cmd/status_sync.go index 69c920ba33..9f3b32141c 100644 --- a/cmd/status_sync.go +++ b/cmd/status_sync.go @@ -15,7 +15,6 @@ import ( "github.com/spf13/cobra" ) -var initialSyncCompleted = regexp.MustCompile(`^\[Sync\] Initial sync completed\. Processed (\d+) changes$`) var syncStopped = regexp.MustCompile(`^\[Sync\] Sync stopped$`) var downstreamChanges = regexp.MustCompile(`^\[Downstream\] Successfully processed (\d+) change\(s\)$`) var upstreamChanges = regexp.MustCompile(`^\[Upstream\] Successfully processed (\d+) change\(s\)$`) @@ -180,12 +179,6 @@ func updateSyncMap(syncMap map[string]*syncStatus, jsonMap map[string]string) er syncMap[identifier].Status = "Error" syncMap[identifier].Error = message syncMap[identifier].LastActivityTime = time - } else if matches := initialSyncCompleted.FindStringSubmatch(message); len(matches) == 2 { - syncMap[identifier].LastActivity = "Initially transferred " + matches[1] + " changes" - syncMap[identifier].LastActivityTime = time - - changes, _ := strconv.Atoi(matches[1]) - syncMap[identifier].TotalChanges += changes } else if matches := downstreamChanges.FindStringSubmatch(message); len(matches) == 2 { syncMap[identifier].LastActivity = "Downloaded " + matches[1] + " changes" syncMap[identifier].LastActivityTime = time diff --git a/cmd/up.go b/cmd/up.go index cc4e644396..08c655903d 100644 --- a/cmd/up.go +++ b/cmd/up.go @@ -156,16 +156,12 @@ func (cmd *UpCmd) Run(cobraCmd *cobra.Command, args []string) { } } - if cmd.flags.deploy || shouldRebuild { + // Check if we find a running release pod + pod, err := getRunningDevSpacePod(cmd.helm, cmd.kubectl) + + if err != nil || cmd.flags.deploy || shouldRebuild { cmd.deployChart() } else { - // Check if we find a running release pod - pod, err := getRunningDevSpacePod(cmd.helm, cmd.kubectl) - - if err != nil { - log.Fatalf("Couldn't find running devspace pod: %s", err.Error()) - } - cmd.pod = pod } diff --git a/pkg/devspace/sync/downstream.go b/pkg/devspace/sync/downstream.go index 2c62f51ce4..b4c2692bae 100644 --- a/pkg/devspace/sync/downstream.go +++ b/pkg/devspace/sync/downstream.go @@ -8,6 +8,7 @@ import ( "os" "os/exec" "path" + "path/filepath" "strconv" "strings" "time" @@ -87,7 +88,9 @@ func (d *downstream) populateFileMap() error { defer d.config.fileIndex.fileMapMutex.Unlock() for _, element := range createFiles { - d.config.fileIndex.fileMap[element.Name] = element + if d.config.fileIndex.fileMap[element.Name] == nil { + d.config.fileIndex.fileMap[element.Name] = element + } } return nil @@ -328,36 +331,28 @@ func (d *downstream) removeFilesAndFolders(removeFiles map[string]*fileInformati d.config.Logf("[Downstream] Remove %d files", numRemoveFiles) } - // A file is only deleted if the following conditions are met: - // - The file name is present in the d.config.fileMap map - // - The file did not change in terms of size and mtime in the d.config.fileMap since we started the collecting changes process - // - The file is present on the filesystem and did not change in terms of size and mtime on the filesystem for key, value := range removeFiles { - if value != nil && fileMap[key] != nil { - // Exclude files on the exclude list - if d.config.downloadIgnoreMatcher != nil { - if d.config.downloadIgnoreMatcher.MatchesPath(key) { - delete(fileMap, key) - continue - } - } + absFilepath := filepath.Join(d.config.WatchPath, key) + if shouldRemoveLocal(absFilepath, value, d.config) { if numRemoveFiles <= 3 { d.config.Logf("[Downstream] Remove %s", key) } - if fileMap[key].IsDirectory { + if value.IsDirectory { deleteSafeRecursive(d.config.WatchPath, key, fileMap, removeFiles, d.config) } else { - if value.Mtime == fileMap[key].Mtime && value.Size == fileMap[key].Size { - if deleteSafe(path.Join(d.config.WatchPath, key), fileMap[key]) == false { - d.config.Logf("[Downstream] Skip file delete %s", key) - } + err := os.Remove(absFilepath) + if err != nil { + d.config.Logf("[Downstream] Skip file delete %s: %v", key, err) + continue } - - delete(fileMap, key) } + } else { + d.config.Logf("[Downstream] Skip delete %s", key) } + + delete(fileMap, key) } } @@ -481,7 +476,6 @@ func (d *downstream) evaluateFile(fileline string, createFiles *[]*fileInformati d.config.fileIndex.fileMapMutex.Lock() defer d.config.fileIndex.fileMapMutex.Unlock() - fileMap := d.config.fileIndex.fileMap fileInformation, err := parseFileInformation(fileline, d.config.DestPath) // Error parsing line @@ -494,59 +488,11 @@ func (d *downstream) evaluateFile(fileline string, createFiles *[]*fileInformati return nil } - // Exclude files on the exclude list - if d.config.ignoreMatcher != nil { - if d.config.ignoreMatcher.MatchesPath(fileInformation.Name) { - return nil - } - } + // File found don't delete it + delete(removeFiles, fileInformation.Name) - // File found, don't delete it - if removeFiles[fileInformation.Name] != nil { - delete(removeFiles, fileInformation.Name) - } - - // Update mode, gid & uid if exists - if fileMap[fileInformation.Name] != nil { - fileMap[fileInformation.Name].RemoteMode = fileInformation.RemoteMode - fileMap[fileInformation.Name].RemoteGID = fileInformation.RemoteGID - fileMap[fileInformation.Name].RemoteUID = fileInformation.RemoteUID - } - - // Exclude files on the exclude list - if d.config.downloadIgnoreMatcher != nil { - if d.config.downloadIgnoreMatcher.MatchesPath(fileInformation.Name) { - return nil - } - } - - // Exclude symlinks - if fileInformation.IsSymbolicLink { - // Add them to the fileMap though - fileMap[fileInformation.Name] = fileInformation - return nil - } - - // Does file already exist in the filemap? - if fileMap[fileInformation.Name] != nil { - // Don't override folders that exist in the filemap - if fileInformation.IsDirectory == false { - // Redownload file if mtime is newer than saved one - if fileInformation.Mtime > fileMap[fileInformation.Name].Mtime { - *createFiles = append(*createFiles, fileInformation) - - return nil - } - - // Redownload file if size changed && file is not older than the one in the fileMap - // the mTime check is necessary, because otherwise we would override older local files that - // are not overridden initially - if fileInformation.Mtime == fileMap[fileInformation.Name].Mtime && fileInformation.Size != fileMap[fileInformation.Name].Size { - *createFiles = append(*createFiles, fileInformation) - } - } - } else { - // We create the file if it doesn't exist in the fileMap + // Should we download the file / folder? + if shouldDownload(fileInformation, d.config) { *createFiles = append(*createFiles, fileInformation) } diff --git a/pkg/devspace/sync/evaluater.go b/pkg/devspace/sync/evaluater.go new file mode 100644 index 0000000000..0658a0a851 --- /dev/null +++ b/pkg/devspace/sync/evaluater.go @@ -0,0 +1,201 @@ +package sync + +import ( + "os" +) + +// s.fileIndex needs to be locked before this function is called +func shouldRemoveRemote(relativePath string, s *SyncConfig) bool { + // Exclude changes on the exclude list + if s.ignoreMatcher != nil { + if s.ignoreMatcher.MatchesPath(relativePath) { + return false + } + } + + // Exclude changes on the upload exclude list + if s.uploadIgnoreMatcher != nil { + if s.uploadIgnoreMatcher.MatchesPath(relativePath) { + return false + } + } + + // File / Folder was already deleted from map so event was already processed or should not be processed + if s.fileIndex.fileMap[relativePath] == nil { + return false + } + + // Exclude symbolic links + if s.fileIndex.fileMap[relativePath].IsSymbolicLink { + return false + } + + return true +} + +// s.fileIndex needs to be locked before this function is called +func shouldUpload(relativePath string, stat os.FileInfo, s *SyncConfig, isInitial bool) bool { + // Exclude if stat is nil + if stat == nil { + return false + } + + // Exclude changes on the exclude list + if s.ignoreMatcher != nil { + if s.ignoreMatcher.MatchesPath(relativePath) { + return false + } + } + + // Exclude changes on the upload exclude list + if s.uploadIgnoreMatcher != nil { + if s.uploadIgnoreMatcher.MatchesPath(relativePath) { + // Add to file map and prevent download if local file is newer than the remote one + if s.fileIndex.fileMap[relativePath] != nil && s.fileIndex.fileMap[relativePath].Mtime < ceilMtime(stat.ModTime()) { + fileInformation := &fileInformation{ + Name: relativePath, + Mtime: ceilMtime(stat.ModTime()), + Size: stat.Size(), + IsDirectory: stat.IsDir(), + } + + // Add it to the fileMap + s.fileIndex.fileMap[relativePath] = fileInformation + } + + return false + } + } + + // Exclude local symlinks + if stat.Mode()&os.ModeSymlink != 0 { + return false + } + + // Check if we already tracked the path + if s.fileIndex.fileMap[relativePath] != nil { + // Folder already exists + if stat.IsDir() { + // We want to initially walk over all files therefore we return true for a directory + // Later on a created directory locally that already exists in the fileMap should be ignored + return isInitial + } + + // Exclude symlinks + if s.fileIndex.fileMap[relativePath].IsSymbolicLink { + return false + } + + if isInitial { + // File is older locally than remote so don't update remote + if ceilMtime(stat.ModTime()) <= s.fileIndex.fileMap[relativePath].Mtime+1 { + return false + } + } else { + // File did not change or was changed by downstream + if ceilMtime(stat.ModTime()) == s.fileIndex.fileMap[relativePath].Mtime && stat.Size() == s.fileIndex.fileMap[relativePath].Size { + return false + } + } + } + + return true +} + +// s.fileIndex needs to be locked before this function is called +func shouldDownload(fileInformation *fileInformation, s *SyncConfig) bool { + // Exclude files on the exclude list + if s.ignoreMatcher != nil { + if s.ignoreMatcher.MatchesPath(fileInformation.Name) { + return false + } + } + + // Update mode, gid & uid if exists + if s.fileIndex.fileMap[fileInformation.Name] != nil { + s.fileIndex.fileMap[fileInformation.Name].RemoteMode = fileInformation.RemoteMode + s.fileIndex.fileMap[fileInformation.Name].RemoteGID = fileInformation.RemoteGID + s.fileIndex.fileMap[fileInformation.Name].RemoteUID = fileInformation.RemoteUID + } + + // Exclude files on the exclude list + if s.downloadIgnoreMatcher != nil { + if s.downloadIgnoreMatcher.MatchesPath(fileInformation.Name) { + return false + } + } + + // Exclude symlinks + if fileInformation.IsSymbolicLink { + // Add them to the fileMap though + s.fileIndex.fileMap[fileInformation.Name] = fileInformation + return false + } + + // Does file already exist in the filemap? + if s.fileIndex.fileMap[fileInformation.Name] != nil { + // Don't override folders that exist in the filemap + if fileInformation.IsDirectory == false { + // Redownload file if mtime is newer than saved one + if fileInformation.Mtime > s.fileIndex.fileMap[fileInformation.Name].Mtime { + return true + } + + // Redownload file if size changed && file is not older than the one in the fileMap + // the mTime check is necessary, because otherwise we would override older local files that + // are not overridden initially + if fileInformation.Mtime == s.fileIndex.fileMap[fileInformation.Name].Mtime && fileInformation.Size != s.fileIndex.fileMap[fileInformation.Name].Size { + return true + } + } + + return false + } + + return true +} + +// s.fileIndex needs to be locked before this function is called +// A file is only deleted if the following conditions are met: +// - The file name is present in the d.config.fileMap map +// - The file did not change in terms of size and mtime in the d.config.fileMap since we started the collecting changes process +// - The file is present on the filesystem and did not change in terms of size and mtime on the filesystem +func shouldRemoveLocal(absFilepath string, fileInformation *fileInformation, s *SyncConfig) bool { + if fileInformation == nil { + return false + } + + // Exclude files on the exclude list + if s.downloadIgnoreMatcher != nil { + if s.downloadIgnoreMatcher.MatchesPath(fileInformation.Name) { + return false + } + } + + // Only delete if mtime and size did not change + stat, err := os.Stat(absFilepath) + if err != nil { + return false + } + + // We don't delete the file if we haven't tracked it + if stat != nil && s.fileIndex.fileMap[fileInformation.Name] != nil { + if stat.IsDir() != s.fileIndex.fileMap[fileInformation.Name].IsDirectory || stat.IsDir() != fileInformation.IsDirectory { + return false + } + + if fileInformation.IsDirectory == false { + // We don't delete the file if it has changed in the map since we collected changes + if fileInformation.Mtime == s.fileIndex.fileMap[fileInformation.Name].Mtime && fileInformation.Size == s.fileIndex.fileMap[fileInformation.Name].Size { + // We don't delete the file if it has changed on the filesystem meanwhile + if ceilMtime(stat.ModTime()) <= fileInformation.Mtime { + return true + } + } + } else { + return true + } + } + + return false +} diff --git a/pkg/devspace/sync/file_information.go b/pkg/devspace/sync/file_information.go index 8f43c4009d..0b274293de 100644 --- a/pkg/devspace/sync/file_information.go +++ b/pkg/devspace/sync/file_information.go @@ -4,6 +4,8 @@ import ( "strconv" "strings" + "github.com/rjeczalik/notify" + "github.com/juju/errors" ) @@ -29,6 +31,22 @@ type fileInformation struct { RemoteGID int // %u } +func (f *fileInformation) Sys() interface{} { + return f +} + +func (f *fileInformation) Path() string { + return f.Name +} + +func (f *fileInformation) Event() notify.Event { + if f.Mtime == 0 { + return notify.Remove + } + + return notify.Create +} + type parsingError struct { msg string } diff --git a/pkg/devspace/sync/sync_config.go b/pkg/devspace/sync/sync_config.go index 248810fcce..4683c4e394 100644 --- a/pkg/devspace/sync/sync_config.go +++ b/pkg/devspace/sync/sync_config.go @@ -4,6 +4,8 @@ import ( "io/ioutil" "os" "path" + "sync" + "time" "github.com/covexo/devspace/pkg/util/log" "github.com/juju/errors" @@ -14,6 +16,7 @@ import ( "k8s.io/client-go/kubernetes" ) +var initialUpstreamBatchSize = 1000 var syncLog log.Logger //StartAck signals to the user that the sync process is starting @@ -47,7 +50,8 @@ type SyncConfig struct { upstream *upstream downstream *downstream - silent bool + silent bool + stopOnce sync.Once // Used for testing testing bool @@ -189,41 +193,49 @@ func (s *SyncConfig) initIgnoreParsers() error { func (s *SyncConfig) mainLoop() { s.Logf("[Sync] Start syncing") - err := s.initialSync() - if err != nil { - s.Error(err) - return - } + // Start upstream as early as possible + go s.startUpstream() - // Start upstream + // Start downstream and do initial sync go func() { defer s.Stop() - // Set up a watchpoint listening for events within a directory tree rooted at specified directory - err := notify.Watch(s.WatchPath+"/...", s.upstream.events, notify.All) - + err := s.initialSync() if err != nil { s.Error(err) return } - defer notify.Stop(s.upstream.events) - - err = s.upstream.mainLoop() - if err != nil { - s.Error(err) - } + s.Logf("[Sync] Initial sync completed") + s.startDownstream() }() +} - // Start downstream - go func() { - defer s.Stop() +func (s *SyncConfig) startUpstream() { + defer s.Stop() - err := s.downstream.mainLoop() - if err != nil { - s.Error(err) - } - }() + // Set up a watchpoint listening for events within a directory tree rooted at specified directory + err := notify.Watch(s.WatchPath+"/...", s.upstream.events, notify.All) + if err != nil { + s.Error(err) + return + } + + defer notify.Stop(s.upstream.events) + + err = s.upstream.mainLoop() + if err != nil { + s.Error(err) + } +} + +func (s *SyncConfig) startDownstream() { + defer s.Stop() + + err := s.downstream.mainLoop() + if err != nil { + s.Error(err) + } } func (s *SyncConfig) initialSync() error { @@ -232,9 +244,10 @@ func (s *SyncConfig) initialSync() error { return errors.Trace(err) } - remoteChanges := make([]*fileInformation, 0, 10) + localChanges := make([]*fileInformation, 0, 10) fileMapClone := make(map[string]*fileInformation) + s.fileIndex.fileMapMutex.Lock() for key, element := range s.fileIndex.fileMap { if element.IsSymbolicLink { continue @@ -242,32 +255,29 @@ func (s *SyncConfig) initialSync() error { fileMapClone[key] = element } + s.fileIndex.fileMapMutex.Unlock() - err = s.diffServerClient(s.WatchPath, &remoteChanges, fileMapClone) + err = s.diffServerClient(s.WatchPath, &localChanges, fileMapClone) if err != nil { return errors.Trace(err) } - if len(remoteChanges) > 0 { - err = s.upstream.applyCreates(remoteChanges) - if err != nil { - return errors.Trace(err) - } + if len(localChanges) > 0 { + go s.sendChangesToUpstream(localChanges) } if len(fileMapClone) > 0 { - localChanges := make([]*fileInformation, 0, len(fileMapClone)) + remoteChanges := make([]*fileInformation, 0, len(fileMapClone)) for _, element := range fileMapClone { - localChanges = append(localChanges, element) + remoteChanges = append(remoteChanges, element) } - err = s.downstream.applyChanges(localChanges, nil) + err = s.downstream.applyChanges(remoteChanges, nil) if err != nil { return errors.Trace(err) } } - s.Logf("[Sync] Initial sync completed. Processed %d changes", len(remoteChanges)+len(fileMapClone)) return nil } @@ -280,50 +290,32 @@ func (s *SyncConfig) diffServerClient(filepath string, sendChanges *[]*fileInfor return nil } - // We skip symlinks - if stat.Mode()&os.ModeSymlink != 0 { - return nil - } - - // Exclude files on the exclude list - if s.ignoreMatcher != nil { - if s.ignoreMatcher.MatchesPath(relativePath) { - return nil - } - } - delete(downloadChanges, relativePath) - // Exclude files on the exclude list - if s.uploadIgnoreMatcher != nil { - if s.uploadIgnoreMatcher.MatchesPath(relativePath) { - return nil - } - } + s.fileIndex.fileMapMutex.Lock() + shouldUpload := shouldUpload(relativePath, stat, s, true) + s.fileIndex.fileMapMutex.Unlock() - // Exclude remote symlinks - if s.fileIndex.fileMap[relativePath] != nil && s.fileIndex.fileMap[relativePath].IsSymbolicLink { + if shouldUpload == false { return nil } if stat.IsDir() { - return s.diffDir(filepath, sendChanges, downloadChanges) + return s.diffDir(filepath, stat, sendChanges, downloadChanges) } - // TODO: Handle the case when local files are older than in the container - if s.fileIndex.fileMap[relativePath] == nil || ceilMtime(stat.ModTime()) > s.fileIndex.fileMap[relativePath].Mtime+1 { - *sendChanges = append(*sendChanges, &fileInformation{ - Name: relativePath, - Mtime: ceilMtime(stat.ModTime()), - Size: stat.Size(), - IsDirectory: false, - }) - } + // Add file to upload + *sendChanges = append(*sendChanges, &fileInformation{ + Name: relativePath, + Mtime: ceilMtime(stat.ModTime()), + Size: stat.Size(), + IsDirectory: false, + }) return nil } -func (s *SyncConfig) diffDir(filepath string, sendChanges *[]*fileInformation, downloadChanges map[string]*fileInformation) error { +func (s *SyncConfig) diffDir(filepath string, stat os.FileInfo, sendChanges *[]*fileInformation, downloadChanges map[string]*fileInformation) error { relativePath := getRelativeFromFullPath(filepath, s.WatchPath) files, err := ioutil.ReadDir(filepath) @@ -333,12 +325,12 @@ func (s *SyncConfig) diffDir(filepath string, sendChanges *[]*fileInformation, d } if len(files) == 0 { - if s.fileIndex.fileMap[relativePath] == nil { - *sendChanges = append(*sendChanges, &fileInformation{ - Name: relativePath, - IsDirectory: true, - }) - } + *sendChanges = append(*sendChanges, &fileInformation{ + Name: relativePath, + Mtime: ceilMtime(stat.ModTime()), + Size: stat.Size(), + IsDirectory: true, + }) } for _, f := range files { @@ -350,12 +342,37 @@ func (s *SyncConfig) diffDir(filepath string, sendChanges *[]*fileInformation, d return nil } -//Stop stops the sync process +func (s *SyncConfig) sendChangesToUpstream(changes []*fileInformation) { + for j := 0; j < len(changes); j += initialUpstreamBatchSize { + // Wait till upstream channel is empty + for len(s.upstream.events) > 0 { + time.Sleep(time.Second) + } + + // Now we send them to upstream + sendBatch := make([]*fileInformation, 0, initialUpstreamBatchSize) + s.fileIndex.fileMapMutex.Lock() + + for i := j; i < (j+initialUpstreamBatchSize) && i < len(changes); i++ { + if s.fileIndex.fileMap[changes[i].Name] == nil || (s.fileIndex.fileMap[changes[i].Name] != nil && changes[i].Mtime > s.fileIndex.fileMap[changes[i].Name].Mtime) { + sendBatch = append(sendBatch, changes[i]) + } + } + + s.fileIndex.fileMapMutex.Unlock() + + // We do this out of the fileIndex lock, because otherwise this could cause a deadlock + // (Upstream waits in getfileInformationFromEvent and upstream.events buffer is full) + for i := 0; i < len(sendBatch); i++ { + s.upstream.events <- sendBatch[i] + } + } +} + +// Stop stops the sync process func (s *SyncConfig) Stop() { - if s.upstream != nil && s.upstream.interrupt != nil { - select { - case <-s.upstream.interrupt: - default: + s.stopOnce.Do(func() { + if s.upstream != nil && s.upstream.interrupt != nil { close(s.upstream.interrupt) if s.upstream.stdinPipe != nil { @@ -371,12 +388,8 @@ func (s *SyncConfig) Stop() { s.upstream.stderrPipe.Close() } } - } - if s.downstream != nil && s.downstream.interrupt != nil { - select { - case <-s.downstream.interrupt: - default: + if s.downstream != nil && s.downstream.interrupt != nil { close(s.downstream.interrupt) if s.downstream.stdinPipe != nil { @@ -392,7 +405,7 @@ func (s *SyncConfig) Stop() { s.downstream.stderrPipe.Close() } } - } - s.Logln("[Sync] Sync stopped") + s.Logln("[Sync] Sync stopped") + }) } diff --git a/pkg/devspace/sync/sync_config_test.go b/pkg/devspace/sync/sync_config_test.go index efedcf4313..49da220843 100644 --- a/pkg/devspace/sync/sync_config_test.go +++ b/pkg/devspace/sync/sync_config_test.go @@ -176,6 +176,8 @@ func TestInitialSync(t *testing.T) { ioutil.WriteFile(path.Join(remote, "testFolder", "testFile3"), []byte(fileContents), 0666) ioutil.WriteFile(path.Join(remote, "testFolder", "testFile4"), []byte(fileContents), 0666) + go syncClient.startUpstream() + // Do initial sync err = syncClient.initialSync() if err != nil { @@ -183,6 +185,9 @@ func TestInitialSync(t *testing.T) { return } + // TODO: Remove sleep and instead wait for upstream changes + time.Sleep(5 * time.Second) + // Check outcome filesToCheck := []string{ "testFile1", @@ -310,7 +315,8 @@ func TestRunningSync(t *testing.T) { } // Start sync and do initial sync - syncClient.mainLoop() + go syncClient.startUpstream() + go syncClient.startDownstream() // Create err = createFileAndWait(remote, local, "2") diff --git a/pkg/devspace/sync/tar.go b/pkg/devspace/sync/tar.go index 60108a0e63..651a2adb78 100644 --- a/pkg/devspace/sync/tar.go +++ b/pkg/devspace/sync/tar.go @@ -71,7 +71,7 @@ func untarNext(tarReader *tar.Reader, destPath, prefix string, config *SyncConfi IsDirectory: stat.IsDir(), } - config.Logf("[Downstream] Don't override %s because file has newer mTime timestamp\n", relativePath) + config.Logf("[Downstream] Don't override %s because file has newer mTime timestamp", relativePath) return true, nil } } diff --git a/pkg/devspace/sync/upstream.go b/pkg/devspace/sync/upstream.go index 25feee23f6..3c73a53cd3 100644 --- a/pkg/devspace/sync/upstream.go +++ b/pkg/devspace/sync/upstream.go @@ -141,80 +141,51 @@ func (u *upstream) getfileInformationFromEvent(events []notify.EventInfo) []*fil changes := make([]*fileInformation, 0, len(events)) for _, event := range events { - fullpath := event.Path() - relativePath := getRelativeFromFullPath(fullpath, u.config.WatchPath) + fileInfo, ok := event.(*fileInformation) - // Exclude changes on the exclude list - if u.config.ignoreMatcher != nil { - if u.config.ignoreMatcher.MatchesPath(relativePath) { - continue - } - } - - // Exclude changes on the upload exclude list - if u.config.uploadIgnoreMatcher != nil { - if u.config.uploadIgnoreMatcher.MatchesPath(relativePath) { - continue - } - } + if ok { + changes = append(changes, fileInfo) + } else { + fullpath := event.Path() + relativePath := getRelativeFromFullPath(fullpath, u.config.WatchPath) - // Determine what kind of change we got (Create or Remove) - newChange := evaluateChange(fileMap, relativePath, fullpath) + // Determine what kind of change we got (Create or Remove) + newChange := evaluateChange(u.config, fileMap, relativePath, fullpath) - if newChange != nil { - changes = append(changes, newChange) + if newChange != nil { + changes = append(changes, newChange) + } } } return changes } -func evaluateChange(fileMap map[string]*fileInformation, relativePath, fullpath string) *fileInformation { +func evaluateChange(s *SyncConfig, fileMap map[string]*fileInformation, relativePath, fullpath string) *fileInformation { stat, err := os.Stat(fullpath) // File / Folder exist -> Create File or Folder // if File / Folder does not exist, we create a new remove change if err == nil { - if fileMap[relativePath] != nil { - // Folder already exists - if stat.IsDir() { - return nil - } - - // File did not change or was changed by downstream - if ceilMtime(stat.ModTime()) == fileMap[relativePath].Mtime && stat.Size() == fileMap[relativePath].Size { - return nil - } - - // Exclude symbolic links - if fileMap[relativePath].IsSymbolicLink { - return nil + if shouldUpload(relativePath, stat, s, false) { + // New Create Task + return &fileInformation{ + Name: relativePath, + Mtime: ceilMtime(stat.ModTime()), + Size: stat.Size(), + IsDirectory: stat.IsDir(), } } - - // New Create Task - return &fileInformation{ - Name: relativePath, - Mtime: ceilMtime(stat.ModTime()), - Size: stat.Size(), - IsDirectory: stat.IsDir(), + } else { + if shouldRemoveRemote(relativePath, s) { + // New Remove Task + return &fileInformation{ + Name: relativePath, + } } } - // File / Folder was already deleted from map so event was already processed or should not be processed - if fileMap[relativePath] == nil { - return nil - } - - // Exclude symbolic links - if fileMap[relativePath].IsSymbolicLink { - return nil - } - - // New Remove Task - return &fileInformation{ - Name: relativePath, - } + return nil } func (u *upstream) applyChanges(changes []*fileInformation) error { diff --git a/pkg/devspace/sync/util.go b/pkg/devspace/sync/util.go index a564567f50..b0e94a0911 100644 --- a/pkg/devspace/sync/util.go +++ b/pkg/devspace/sync/util.go @@ -241,67 +241,44 @@ func deleteSafeRecursive(basepath, relativePath string, fileMap map[string]*file // We don't delete the folder or the contents if we haven't tracked it if fileMap[relativePath] == nil || removeFiles[relativePath] == nil { config.Logf("[Downstream] Skip delete directory %s\n", relativePath) - return } // Delete directory from fileMap defer delete(fileMap, relativePath) files, err := ioutil.ReadDir(absolutePath) - if err != nil { return } for _, f := range files { - if f.IsDir() { - deleteSafeRecursive(basepath, path.Join(relativePath, f.Name()), fileMap, removeFiles, config) - } else { - filepath := path.Join(relativePath, f.Name()) - fileDeleted := false - - // We don't delete the file if we haven't tracked it - if fileMap[filepath] != nil && removeFiles[filepath] != nil { - // We don't delete the file if it has changed in the map since we collected changes - if removeFiles[filepath].Mtime == fileMap[filepath].Mtime && removeFiles[filepath].Size == fileMap[filepath].Size { - // We don't delete the file if it has changed on the filesystem meanwhile - fileDeleted = deleteSafe(path.Join(basepath, filepath), fileMap[filepath]) - } - } + filepath := path.Join(relativePath, f.Name()) + absFilepath := path.Join(basepath, filepath) - if fileDeleted == false { - config.Logf("[Downstream] Skip file delete %s\n", relativePath) + if shouldRemoveLocal(absFilepath, fileMap[filepath], config) { + if f.IsDir() { + deleteSafeRecursive(basepath, filepath, fileMap, removeFiles, config) } else { - delete(fileMap, filepath) + err = os.Remove(absFilepath) + if err != nil { + config.Logf("[Downstream] Skip file delete %s: %v", relativePath, err) + continue + } } + } else { + config.Logf("[Downstream] Skip delete %s", relativePath) } + + delete(fileMap, filepath) } // This will not remove the directory if there is still a file or directory in it err = os.Remove(absolutePath) - if err != nil { config.Logf("[Downstream] Skip delete directory %s, because %s\n", relativePath, err.Error()) } } -func deleteSafe(path string, fileInformation *fileInformation) bool { - // Only delete if mtime and size did not change - stat, err := os.Stat(path) - - // TODO: uncomment this line for more safety (However we have to change the initial sync functionality that older files locally are either uplaoded or the newer files on the server downloaded) - // if err == nil && stat.Size() == fileInformation.Size && ceilMtime(stat.ModTime()) == fileInformation.Mtime { - if err == nil && ceilMtime(stat.ModTime()) <= fileInformation.Mtime { - err = os.Remove(path) - - if err == nil { - return true - } - } - - return false -} - func compilePaths(excludePaths []string) (gitignore.IgnoreParser, error) { if len(excludePaths) > 0 { ignoreParser, err := gitignore.CompileIgnoreLines(excludePaths...)