Sync improvements

cmd/status_sync.go

@@ -15,7 +15,6 @@ import (
 	"github.com/spf13/cobra"
 )
 
-var initialSyncCompleted = regexp.MustCompile(`^\[Sync\] Initial sync completed\. Processed (\d+) changes$`)
 var syncStopped = regexp.MustCompile(`^\[Sync\] Sync stopped$`)
 var downstreamChanges = regexp.MustCompile(`^\[Downstream\] Successfully processed (\d+) change\(s\)$`)
 var upstreamChanges = regexp.MustCompile(`^\[Upstream\] Successfully processed (\d+) change\(s\)$`)
@@ -180,12 +179,6 @@ func updateSyncMap(syncMap map[string]*syncStatus, jsonMap map[string]string) er
 		syncMap[identifier].Status = "Error"
 		syncMap[identifier].Error = message
 		syncMap[identifier].LastActivityTime = time
-	} else if matches := initialSyncCompleted.FindStringSubmatch(message); len(matches) == 2 {
-		syncMap[identifier].LastActivity = "Initially transferred " + matches[1] + " changes"
-		syncMap[identifier].LastActivityTime = time
-
-		changes, _ := strconv.Atoi(matches[1])
-		syncMap[identifier].TotalChanges += changes
 	} else if matches := downstreamChanges.FindStringSubmatch(message); len(matches) == 2 {
 		syncMap[identifier].LastActivity = "Downloaded " + matches[1] + " changes"
 		syncMap[identifier].LastActivityTime = time

cmd/up.go

@@ -156,16 +156,12 @@ func (cmd *UpCmd) Run(cobraCmd *cobra.Command, args []string) {
 		}
 	}
 
-	if cmd.flags.deploy || shouldRebuild {
+	// Check if we find a running release pod
+	pod, err := getRunningDevSpacePod(cmd.helm, cmd.kubectl)
+
+	if err != nil || cmd.flags.deploy || shouldRebuild {
 		cmd.deployChart()
 	} else {
-		// Check if we find a running release pod
-		pod, err := getRunningDevSpacePod(cmd.helm, cmd.kubectl)
-
-		if err != nil {
-			log.Fatalf("Couldn't find running devspace pod: %s", err.Error())
-		}
-
 		cmd.pod = pod
 	}
 

pkg/devspace/sync/downstream.go

@@ -8,6 +8,7 @@ import (
 	"os"
 	"os/exec"
 	"path"
+	"path/filepath"
 	"strconv"
 	"strings"
 	"time"
@@ -87,7 +88,9 @@ func (d *downstream) populateFileMap() error {
 	defer d.config.fileIndex.fileMapMutex.Unlock()
 
 	for _, element := range createFiles {
-		d.config.fileIndex.fileMap[element.Name] = element
+		if d.config.fileIndex.fileMap[element.Name] == nil {
+			d.config.fileIndex.fileMap[element.Name] = element
+		}
 	}
 
 	return nil
@@ -328,36 +331,28 @@ func (d *downstream) removeFilesAndFolders(removeFiles map[string]*fileInformati
 		d.config.Logf("[Downstream] Remove %d files", numRemoveFiles)
 	}
 
-	// A file is only deleted if the following conditions are met:
-	// - The file name is present in the d.config.fileMap map
-	// - The file did not change in terms of size and mtime in the d.config.fileMap since we started the collecting changes process
-	// - The file is present on the filesystem and did not change in terms of size and mtime on the filesystem
 	for key, value := range removeFiles {
-		if value != nil && fileMap[key] != nil {
-			// Exclude files on the exclude list
-			if d.config.downloadIgnoreMatcher != nil {
-				if d.config.downloadIgnoreMatcher.MatchesPath(key) {
-					delete(fileMap, key)
-					continue
-				}
-			}
+		absFilepath := filepath.Join(d.config.WatchPath, key)
 
+		if shouldRemoveLocal(absFilepath, value, d.config) {
 			if numRemoveFiles <= 3 {
 				d.config.Logf("[Downstream] Remove %s", key)
 			}
 
-			if fileMap[key].IsDirectory {
+			if value.IsDirectory {
 				deleteSafeRecursive(d.config.WatchPath, key, fileMap, removeFiles, d.config)
 			} else {
-				if value.Mtime == fileMap[key].Mtime && value.Size == fileMap[key].Size {
-					if deleteSafe(path.Join(d.config.WatchPath, key), fileMap[key]) == false {
-						d.config.Logf("[Downstream] Skip file delete %s", key)
-					}
+				err := os.Remove(absFilepath)
+				if err != nil {
+					d.config.Logf("[Downstream] Skip file delete %s: %v", key, err)
+					continue
 				}
-
-				delete(fileMap, key)
 			}
+		} else {
+			d.config.Logf("[Downstream] Skip delete %s", key)
 		}
+
+		delete(fileMap, key)
 	}
 }
 
@@ -481,7 +476,6 @@ func (d *downstream) evaluateFile(fileline string, createFiles *[]*fileInformati
 	d.config.fileIndex.fileMapMutex.Lock()
 	defer d.config.fileIndex.fileMapMutex.Unlock()
 
-	fileMap := d.config.fileIndex.fileMap
 	fileInformation, err := parseFileInformation(fileline, d.config.DestPath)
 
 	// Error parsing line
@@ -494,59 +488,11 @@ func (d *downstream) evaluateFile(fileline string, createFiles *[]*fileInformati
 		return nil
 	}
 
-	// Exclude files on the exclude list
-	if d.config.ignoreMatcher != nil {
-		if d.config.ignoreMatcher.MatchesPath(fileInformation.Name) {
-			return nil
-		}
-	}
+	// File found don't delete it
+	delete(removeFiles, fileInformation.Name)
 
-	// File found, don't delete it
-	if removeFiles[fileInformation.Name] != nil {
-		delete(removeFiles, fileInformation.Name)
-	}
-
-	// Update mode, gid & uid if exists
-	if fileMap[fileInformation.Name] != nil {
-		fileMap[fileInformation.Name].RemoteMode = fileInformation.RemoteMode
-		fileMap[fileInformation.Name].RemoteGID = fileInformation.RemoteGID
-		fileMap[fileInformation.Name].RemoteUID = fileInformation.RemoteUID
-	}
-
-	// Exclude files on the exclude list
-	if d.config.downloadIgnoreMatcher != nil {
-		if d.config.downloadIgnoreMatcher.MatchesPath(fileInformation.Name) {
-			return nil
-		}
-	}
-
-	// Exclude symlinks
-	if fileInformation.IsSymbolicLink {
-		// Add them to the fileMap though
-		fileMap[fileInformation.Name] = fileInformation
-		return nil
-	}
-
-	// Does file already exist in the filemap?
-	if fileMap[fileInformation.Name] != nil {
-		// Don't override folders that exist in the filemap
-		if fileInformation.IsDirectory == false {
-			// Redownload file if mtime is newer than saved one
-			if fileInformation.Mtime > fileMap[fileInformation.Name].Mtime {
-				*createFiles = append(*createFiles, fileInformation)
-
-				return nil
-			}
-
-			// Redownload file if size changed && file is not older than the one in the fileMap
-			// the mTime check is necessary, because otherwise we would override older local files that
-			// are not overridden initially
-			if fileInformation.Mtime == fileMap[fileInformation.Name].Mtime && fileInformation.Size != fileMap[fileInformation.Name].Size {
-				*createFiles = append(*createFiles, fileInformation)
-			}
-		}
-	} else {
-		// We create the file if it doesn't exist in the fileMap
+	// Should we download the file / folder?
+	if shouldDownload(fileInformation, d.config) {
 		*createFiles = append(*createFiles, fileInformation)
 	}
 

pkg/devspace/sync/evaluater.go

@@ -0,0 +1,201 @@
+package sync
+
+import (
+	"os"
+)
+
+// s.fileIndex needs to be locked before this function is called
+func shouldRemoveRemote(relativePath string, s *SyncConfig) bool {
+	// Exclude changes on the exclude list
+	if s.ignoreMatcher != nil {
+		if s.ignoreMatcher.MatchesPath(relativePath) {
+			return false
+		}
+	}
+
+	// Exclude changes on the upload exclude list
+	if s.uploadIgnoreMatcher != nil {
+		if s.uploadIgnoreMatcher.MatchesPath(relativePath) {
+			return false
+		}
+	}
+
+	// File / Folder was already deleted from map so event was already processed or should not be processed
+	if s.fileIndex.fileMap[relativePath] == nil {
+		return false
+	}
+
+	// Exclude symbolic links
+	if s.fileIndex.fileMap[relativePath].IsSymbolicLink {
+		return false
+	}
+
+	return true
+}
+
+// s.fileIndex needs to be locked before this function is called
+func shouldUpload(relativePath string, stat os.FileInfo, s *SyncConfig, isInitial bool) bool {
+	// Exclude if stat is nil
+	if stat == nil {
+		return false
+	}
+
+	// Exclude changes on the exclude list
+	if s.ignoreMatcher != nil {
+		if s.ignoreMatcher.MatchesPath(relativePath) {
+			return false
+		}
+	}
+
+	// Exclude changes on the upload exclude list
+	if s.uploadIgnoreMatcher != nil {
+		if s.uploadIgnoreMatcher.MatchesPath(relativePath) {
+			// Add to file map and prevent download if local file is newer than the remote one
+			if s.fileIndex.fileMap[relativePath] != nil && s.fileIndex.fileMap[relativePath].Mtime < ceilMtime(stat.ModTime()) {
+				fileInformation := &fileInformation{
+					Name:        relativePath,
+					Mtime:       ceilMtime(stat.ModTime()),
+					Size:        stat.Size(),
+					IsDirectory: stat.IsDir(),
+				}
+
+				// Add it to the fileMap
+				s.fileIndex.fileMap[relativePath] = fileInformation
+			}
+
+			return false
+		}
+	}
+
+	// Exclude local symlinks
+	if stat.Mode()&os.ModeSymlink != 0 {
+		return false
+	}
+
+	// Check if we already tracked the path
+	if s.fileIndex.fileMap[relativePath] != nil {
+		// Folder already exists
+		if stat.IsDir() {
+			// We want to initially walk over all files therefore we return true for a directory
+			// Later on a created directory locally that already exists in the fileMap should be ignored
+			return isInitial
+		}
+
+		// Exclude symlinks
+		if s.fileIndex.fileMap[relativePath].IsSymbolicLink {
+			return false
+		}
+
+		if isInitial {
+			// File is older locally than remote so don't update remote
+			if ceilMtime(stat.ModTime()) <= s.fileIndex.fileMap[relativePath].Mtime+1 {
+				return false
+			}
+		} else {
+			// File did not change or was changed by downstream
+			if ceilMtime(stat.ModTime()) == s.fileIndex.fileMap[relativePath].Mtime && stat.Size() == s.fileIndex.fileMap[relativePath].Size {
+				return false
+			}
+		}
+	}
+
+	return true
+}
+
+// s.fileIndex needs to be locked before this function is called
+func shouldDownload(fileInformation *fileInformation, s *SyncConfig) bool {
+	// Exclude files on the exclude list
+	if s.ignoreMatcher != nil {
+		if s.ignoreMatcher.MatchesPath(fileInformation.Name) {
+			return false
+		}
+	}
+
+	// Update mode, gid & uid if exists
+	if s.fileIndex.fileMap[fileInformation.Name] != nil {
+		s.fileIndex.fileMap[fileInformation.Name].RemoteMode = fileInformation.RemoteMode
+		s.fileIndex.fileMap[fileInformation.Name].RemoteGID = fileInformation.RemoteGID
+		s.fileIndex.fileMap[fileInformation.Name].RemoteUID = fileInformation.RemoteUID
+	}
+
+	// Exclude files on the exclude list
+	if s.downloadIgnoreMatcher != nil {
+		if s.downloadIgnoreMatcher.MatchesPath(fileInformation.Name) {
+			return false
+		}
+	}
+
+	// Exclude symlinks
+	if fileInformation.IsSymbolicLink {
+		// Add them to the fileMap though
+		s.fileIndex.fileMap[fileInformation.Name] = fileInformation
+		return false
+	}
+
+	// Does file already exist in the filemap?
+	if s.fileIndex.fileMap[fileInformation.Name] != nil {
+		// Don't override folders that exist in the filemap
+		if fileInformation.IsDirectory == false {
+			// Redownload file if mtime is newer than saved one
+			if fileInformation.Mtime > s.fileIndex.fileMap[fileInformation.Name].Mtime {
+				return true
+			}
+
+			// Redownload file if size changed && file is not older than the one in the fileMap
+			// the mTime check is necessary, because otherwise we would override older local files that
+			// are not overridden initially
+			if fileInformation.Mtime == s.fileIndex.fileMap[fileInformation.Name].Mtime && fileInformation.Size != s.fileIndex.fileMap[fileInformation.Name].Size {
+				return true
+			}
+		}
+
+		return false
+	}
+
+	return true
+}
+
+// s.fileIndex needs to be locked before this function is called
+// A file is only deleted if the following conditions are met:
+// - The file name is present in the d.config.fileMap map
+// - The file did not change in terms of size and mtime in the d.config.fileMap since we started the collecting changes process
+// - The file is present on the filesystem and did not change in terms of size and mtime on the filesystem
+func shouldRemoveLocal(absFilepath string, fileInformation *fileInformation, s *SyncConfig) bool {
+	if fileInformation == nil {
+		return false
+	}
+
+	// Exclude files on the exclude list
+	if s.downloadIgnoreMatcher != nil {
+		if s.downloadIgnoreMatcher.MatchesPath(fileInformation.Name) {
+			return false
+		}
+	}
+
+	// Only delete if mtime and size did not change
+	stat, err := os.Stat(absFilepath)
+	if err != nil {
+		return false
+	}
+
+	// We don't delete the file if we haven't tracked it
+	if stat != nil && s.fileIndex.fileMap[fileInformation.Name] != nil {
+		if stat.IsDir() != s.fileIndex.fileMap[fileInformation.Name].IsDirectory || stat.IsDir() != fileInformation.IsDirectory {
+			return false
+		}
+
+		if fileInformation.IsDirectory == false {
+			// We don't delete the file if it has changed in the map since we collected changes
+			if fileInformation.Mtime == s.fileIndex.fileMap[fileInformation.Name].Mtime && fileInformation.Size == s.fileIndex.fileMap[fileInformation.Name].Size {
+				// We don't delete the file if it has changed on the filesystem meanwhile
+				if ceilMtime(stat.ModTime()) <= fileInformation.Mtime {
+					return true
+				}
+			}
+		} else {
+			return true
+		}
+	}
+
+	return false
+}