diff --git a/pkg/devspace/sync/downstream.go b/pkg/devspace/sync/downstream.go index cc64f3aae7..b3a4e2f79a 100644 --- a/pkg/devspace/sync/downstream.go +++ b/pkg/devspace/sync/downstream.go @@ -109,7 +109,6 @@ func (d *downstream) mainLoop() error { } amountChanges := len(createFiles) + len(removeFiles) - if lastAmountChanges > 0 && amountChanges == lastAmountChanges { err = d.applyChanges(createFiles, removeFiles) if err != nil { @@ -150,6 +149,139 @@ func (d *downstream) cloneFileMap() map[string]*fileInformation { return mapClone } +func (d *downstream) collectChanges(removeFiles map[string]*fileInformation) ([]*fileInformation, error) { + createFiles := make([]*fileInformation, 0, 128) + destPathFound := false + + // Write find command to stdin pipe + cmd := getFindCommand(d.config.DestPath) + _, err := d.stdinPipe.Write([]byte(cmd)) + if err != nil { + return nil, errors.Trace(err) + } + + buf := make([]byte, 0, 512) + overlap := "" + done := false + + for done == false { + n, err := d.stdoutPipe.Read(buf[:cap(buf)]) + buf = buf[:n] + + if n == 0 { + if err == nil { + continue + } + if err == io.EOF { + return nil, errors.Trace(fmt.Errorf("[Downstream] Stream closed unexpectedly")) + } + + return nil, errors.Trace(err) + } + + // Error reading from stdout + if err != nil && err != io.EOF { + return nil, errors.Trace(err) + } + + done, overlap, err = d.parseLines(string(buf), overlap, &createFiles, removeFiles, &destPathFound) + if err != nil { + if _, ok := err.(parsingError); ok { + time.Sleep(time.Second * 4) + return d.collectChanges(removeFiles) + } + + // No trace here because it could be a parsing error + return nil, errors.Trace(err) + } + } + + if destPathFound == false { + return nil, errors.New("DestPath not found, find command did not execute correctly") + } + + return createFiles, nil +} + +func (d *downstream) parseLines(buffer, overlap string, createFiles *[]*fileInformation, removeFiles map[string]*fileInformation, destPathFound *bool) (bool, string, error) { + lines := strings.Split(buffer, "\n") + + for index, element := range lines { + line := "" + + if index == 0 { + if len(lines) > 1 { + line = overlap + element + } else { + overlap += element + } + } else if index == len(lines)-1 { + overlap = element + } else { + line = element + } + + if line == EndAck || overlap == EndAck { + return true, overlap, nil + } else if line == ErrorAck || overlap == ErrorAck { + return true, "", parsingError{ + msg: "Parsing Error", + } + } else if line != "" { + destPath, err := d.evaluateFile(line, createFiles, removeFiles) + if destPath { + *destPathFound = destPath + } + + if err != nil { + return true, "", errors.Trace(err) + } + } + } + + return false, overlap, nil +} + +func (d *downstream) evaluateFile(fileline string, createFiles *[]*fileInformation, removeFiles map[string]*fileInformation) (bool, error) { + d.config.fileIndex.fileMapMutex.Lock() + defer d.config.fileIndex.fileMapMutex.Unlock() + + fileInformation, err := parseFileInformation(fileline, d.config.DestPath) + + // Error parsing line + if err != nil { + return false, errors.Trace(err) + } + + // No file found + if fileInformation == nil { + return true, nil + } + + // File found don't delete it + delete(removeFiles, fileInformation.Name) + + // Update mode, gid & uid if exists + if d.config.fileIndex.fileMap[fileInformation.Name] != nil { + d.config.fileIndex.fileMap[fileInformation.Name].RemoteMode = fileInformation.RemoteMode + d.config.fileIndex.fileMap[fileInformation.Name].RemoteGID = fileInformation.RemoteGID + d.config.fileIndex.fileMap[fileInformation.Name].RemoteUID = fileInformation.RemoteUID + } + + // Exclude symlinks + if fileInformation.IsSymbolicLink { + // Add them to the fileMap though + d.config.fileIndex.fileMap[fileInformation.Name] = fileInformation + } + + // Should we download the file / folder? + if shouldDownload(fileInformation, d.config) { + *createFiles = append(*createFiles, fileInformation) + } + + return false, nil +} + func (d *downstream) applyChanges(createFiles []*fileInformation, removeFiles map[string]*fileInformation) error { var err error @@ -384,129 +516,3 @@ func (d *downstream) createFolders(createFolders []*fileInformation) { } } } - -func (d *downstream) collectChanges(removeFiles map[string]*fileInformation) ([]*fileInformation, error) { - createFiles := make([]*fileInformation, 0, 128) - - // Write find command to stdin pipe - cmd := getFindCommand(d.config.DestPath) - _, err := d.stdinPipe.Write([]byte(cmd)) - if err != nil { - return nil, errors.Trace(err) - } - - buf := make([]byte, 0, 512) - overlap := "" - done := false - - for done == false { - n, err := d.stdoutPipe.Read(buf[:cap(buf)]) - buf = buf[:n] - - if n == 0 { - if err == nil { - continue - } - - if err == io.EOF { - return nil, errors.Trace(fmt.Errorf("[Downstream] Stream closed unexpectedly")) - } - - return nil, errors.Trace(err) - } - - // Error reading from stdout - if err != nil && err != io.EOF { - return nil, errors.Trace(err) - } - - done, overlap, err = d.parseLines(string(buf), overlap, &createFiles, removeFiles) - if err != nil { - if _, ok := err.(parsingError); ok { - time.Sleep(time.Second * 4) - return d.collectChanges(removeFiles) - } - - // No trace here because it could be a parsing error - return nil, errors.Trace(err) - } - } - - return createFiles, nil -} - -func (d *downstream) parseLines(buffer, overlap string, createFiles *[]*fileInformation, removeFiles map[string]*fileInformation) (bool, string, error) { - lines := strings.Split(buffer, "\n") - - for index, element := range lines { - line := "" - - if index == 0 { - if len(lines) > 1 { - line = overlap + element - } else { - overlap += element - } - } else if index == len(lines)-1 { - overlap = element - } else { - line = element - } - - if line == EndAck || overlap == EndAck { - return true, overlap, nil - } else if line == ErrorAck || overlap == ErrorAck { - return true, "", parsingError{ - msg: "Parsing Error", - } - } else if line != "" { - err := d.evaluateFile(line, createFiles, removeFiles) - - if err != nil { - return true, "", errors.Trace(err) - } - } - } - - return false, overlap, nil -} - -func (d *downstream) evaluateFile(fileline string, createFiles *[]*fileInformation, removeFiles map[string]*fileInformation) error { - d.config.fileIndex.fileMapMutex.Lock() - defer d.config.fileIndex.fileMapMutex.Unlock() - - fileInformation, err := parseFileInformation(fileline, d.config.DestPath) - - // Error parsing line - if err != nil { - return errors.Trace(err) - } - - // No file found - if fileInformation == nil { - return nil - } - - // File found don't delete it - delete(removeFiles, fileInformation.Name) - - // Update mode, gid & uid if exists - if d.config.fileIndex.fileMap[fileInformation.Name] != nil { - d.config.fileIndex.fileMap[fileInformation.Name].RemoteMode = fileInformation.RemoteMode - d.config.fileIndex.fileMap[fileInformation.Name].RemoteGID = fileInformation.RemoteGID - d.config.fileIndex.fileMap[fileInformation.Name].RemoteUID = fileInformation.RemoteUID - } - - // Exclude symlinks - if fileInformation.IsSymbolicLink { - // Add them to the fileMap though - d.config.fileIndex.fileMap[fileInformation.Name] = fileInformation - } - - // Should we download the file / folder? - if shouldDownload(fileInformation, d.config) { - *createFiles = append(*createFiles, fileInformation) - } - - return nil -} diff --git a/pkg/devspace/sync/file_information.go b/pkg/devspace/sync/file_information.go index 5c7d90833c..b72bcadae8 100644 --- a/pkg/devspace/sync/file_information.go +++ b/pkg/devspace/sync/file_information.go @@ -56,18 +56,16 @@ func (p parsingError) Error() string { } func getFindCommand(destPath string) string { - return "mkdir -p '" + destPath + "' && find -L '" + destPath + "' -exec stat -c \"%n///%s,%Y,%f,%a,%u,%g\" {} + 2>/dev/null && echo -n \"" + EndAck + "\" || echo \"" + ErrorAck + "\"\n" + return "mkdir -p '" + destPath + "' && find -L '" + destPath + "' -exec stat -c \"%n///%s,%Y,%f,%a,%u,%g\" {} + 2>/dev/null && echo -n \"" + EndAck + "\" || echo -n \"" + ErrorAck + "\"\n" } func parseFileInformation(fileline, destPath string) (*fileInformation, error) { fileinfo := fileInformation{} t := strings.Split(fileline, "///") - if len(t) != 2 { return nil, errors.New("[Downstream] Wrong fileline: " + fileline) } - if len(t[0]) <= len(destPath) { return nil, nil } @@ -75,13 +73,11 @@ func parseFileInformation(fileline, destPath string) (*fileInformation, error) { fileinfo.Name = t[0][len(destPath):] t = strings.Split(t[1], ",") - if len(t) != 6 { return nil, errors.New("[Downstream] Wrong fileline: " + fileline) } size, err := strconv.Atoi(t[0]) - if err != nil { return nil, errors.Trace(err) } @@ -89,7 +85,6 @@ func parseFileInformation(fileline, destPath string) (*fileInformation, error) { fileinfo.Size = int64(size) mTime, err := strconv.Atoi(t[1]) - if err != nil { return nil, errors.Trace(err) } @@ -97,7 +92,6 @@ func parseFileInformation(fileline, destPath string) (*fileInformation, error) { fileinfo.Mtime = int64(mTime) rawMode, err := strconv.ParseUint(t[2], 16, 32) // Parse hex string into uint64 - if err != nil { return nil, errors.Trace(err) } @@ -107,7 +101,6 @@ func parseFileInformation(fileline, destPath string) (*fileInformation, error) { fileinfo.IsDirectory = (rawMode & IsDirectory) == IsDirectory mode, err := strconv.ParseInt(t[3], 8, 32) - if err != nil { return nil, errors.Trace(err) } @@ -115,7 +108,6 @@ func parseFileInformation(fileline, destPath string) (*fileInformation, error) { fileinfo.RemoteMode = mode uid, err := strconv.Atoi(t[4]) - if err != nil { return nil, errors.Trace(err) } @@ -123,7 +115,6 @@ func parseFileInformation(fileline, destPath string) (*fileInformation, error) { fileinfo.RemoteUID = uid gid, err := strconv.Atoi(t[5]) - if err != nil { return nil, errors.Trace(err) }