From fd695c54ddd82665737f6fa79d7cfb367f350ba7 Mon Sep 17 00:00:00 2001 From: zsolt-marta-bitrise Date: Tue, 10 Sep 2024 12:20:10 +0200 Subject: [PATCH 01/10] feat: ACI-2805 Add flag to follow xcode symlinks --- cmd/saveXcodeDerivedDataFiles.go | 30 +++++++- cmd/saveXcodeDerivedDataFiles_test.go | 2 +- internal/xcode/file_group_info.go | 102 ++++++++++++++++++++++---- internal/xcode/metadata.go | 19 ++++- 4 files changed, 130 insertions(+), 23 deletions(-) diff --git a/cmd/saveXcodeDerivedDataFiles.go b/cmd/saveXcodeDerivedDataFiles.go index 706b5738..4591b968 100644 --- a/cmd/saveXcodeDerivedDataFiles.go +++ b/cmd/saveXcodeDerivedDataFiles.go @@ -38,6 +38,7 @@ var saveXcodeDerivedDataFilesCmd = &cobra.Command{ cacheKey, _ := cmd.Flags().GetString("key") ddPath, _ := cmd.Flags().GetString("deriveddata-path") xcodeCachePath, _ := cmd.Flags().GetString("xcodecache-path") + followSymlinks, _ := cmd.Flags().GetBool("follow-symlinks") tracker := xcode.NewDefaultStepTracker("save-xcode-build-cache", os.Getenv, logger) defer tracker.Wait() @@ -49,7 +50,18 @@ var saveXcodeDerivedDataFilesCmd = &cobra.Command{ return fmt.Errorf("read auth config from environments: %w", err) } - op, cmdError := saveXcodeDerivedDataFilesCmdFn(cmd.Context(), authConfig, CacheMetadataPath, projectRoot, cacheKey, ddPath, xcodeCachePath, logger, tracker, startT, os.Getenv) + op, cmdError := saveXcodeDerivedDataFilesCmdFn(cmd.Context(), + authConfig, + CacheMetadataPath, + projectRoot, + cacheKey, + ddPath, + xcodeCachePath, + followSymlinks, + logger, + tracker, + startT, + os.Getenv) if op != nil { if cmdError != nil { errStr := cmdError.Error() @@ -86,10 +98,21 @@ func init() { panic(err) } saveXcodeDerivedDataFilesCmd.Flags().String("xcodecache-path", "", "Path to the Xcode cache directory folder to be saved. If not set, it will not be uploaded.") + saveXcodeDerivedDataFilesCmd.Flags().Bool("follow-symlinks", false, "Follow symlinks when calculating metadata (default: false)") } -func saveXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.CacheAuthConfig, cacheMetadataPath, projectRoot, providedCacheKey, derivedDataPath, xcodeCachePath string, - logger log.Logger, tracker xcode.StepAnalyticsTracker, startT time.Time, envProvider func(string) string) (*xa.CacheOperation, error) { +func saveXcodeDerivedDataFilesCmdFn(ctx context.Context, + authConfig common.CacheAuthConfig, + cacheMetadataPath, + projectRoot, + providedCacheKey, + derivedDataPath, + xcodeCachePath string, + followSymlinks bool, + logger log.Logger, + tracker xcode.StepAnalyticsTracker, + startT time.Time, + envProvider func(string) string) (*xa.CacheOperation, error) { var err error var cacheKey string if providedCacheKey == "" { @@ -126,6 +149,7 @@ func saveXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Cache DerivedDataPath: derivedDataPath, XcodeCacheDirPath: xcodeCachePath, CacheKey: cacheKey, + FollowSymlinks: followSymlinks, }, envProvider, logger) if err != nil { return op, fmt.Errorf("create metadata: %w", err) diff --git a/cmd/saveXcodeDerivedDataFiles_test.go b/cmd/saveXcodeDerivedDataFiles_test.go index ae7dd8c6..365a13ac 100644 --- a/cmd/saveXcodeDerivedDataFiles_test.go +++ b/cmd/saveXcodeDerivedDataFiles_test.go @@ -36,7 +36,7 @@ func Test_saveXcodeDerivedDataFilesCmdFn(t *testing.T) { envVars := createEnvProvider(map[string]string{ "BITRISEIO_BITRISE_SERVICES_ACCESS_TOKEN": "ServiceAccessTokenValue", }) - _, err := saveXcodeDerivedDataFilesCmdFn(context.Background(), common.CacheAuthConfig{}, "", "", "", "", "", mockLogger, mockTracker, time.Now(), envVars) + _, err := saveXcodeDerivedDataFilesCmdFn(context.Background(), common.CacheAuthConfig{}, "", "", "", "", "", false, mockLogger, mockTracker, time.Now(), envVars) // then require.EqualError(t, err, "get cache key: cache key is required if BITRISE_GIT_BRANCH env var is not set") diff --git a/internal/xcode/file_group_info.go b/internal/xcode/file_group_info.go index a0293c8c..1aae7844 100644 --- a/internal/xcode/file_group_info.go +++ b/internal/xcode/file_group_info.go @@ -58,7 +58,7 @@ func (mc *fileGroupInfoCollector) AddDir(dirInfo *DirectoryInfo) { mc.Dirs = append(mc.Dirs, dirInfo) } -func collectFileGroupInfo(cacheDirPath string, rootDir string, collectAttributes bool, logger log.Logger) (FileGroupInfo, error) { +func collectFileGroupInfo(cacheDirPath string, rootDir string, collectAttributes, followSymlinks bool, logger log.Logger) (FileGroupInfo, error) { var dd FileGroupInfo fgi := fileGroupInfoCollector{ @@ -80,7 +80,14 @@ func collectFileGroupInfo(cacheDirPath string, rootDir string, collectAttributes defer wg.Done() defer func() { <-semaphore }() // Release a slot in the semaphore - if err := collectFileMetadata(path, d, &fgi, collectAttributes, rootDir, logger); err != nil { + inf, err := d.Info() + if err != nil { + logger.Errorf("get file info: %v", err) + + return + } + + if err := collectFileMetadata(path, inf, inf.IsDir(), &fgi, collectAttributes, followSymlinks, rootDir, logger); err != nil { logger.Errorf("Failed to collect metadata: %s", err) } }(d) @@ -103,29 +110,60 @@ func collectFileGroupInfo(cacheDirPath string, rootDir string, collectAttributes return dd, nil } -func collectFileMetadata(path string, d fs.DirEntry, fgi *fileGroupInfoCollector, collectAttributes bool, rootDir string, logger log.Logger) error { - absPath, err := filepath.Abs(path) +// nolint:wrapcheck +func collectSymlink(path string, fgi *fileGroupInfoCollector, followSymlinks bool, rootDir string, logger log.Logger) error { + if !followSymlinks { + logger.Debugf("Skipping symbolic link: %s", path) + + return nil + } + + target, err := resolveSymlink(path) if err != nil { - return fmt.Errorf("get absolute path: %w", err) + return fmt.Errorf("resolve symlink %s: %w", path, err) } - inf, err := d.Info() + logger.Debugf("Resolved symlink %s to %s", path, target) + + stat, err := os.Stat(target) if err != nil { - return fmt.Errorf("get file info: %w", err) + return fmt.Errorf("stat target: %w", err) } + if !stat.IsDir() { + return collectFileMetadata(target, stat, false, fgi, false, followSymlinks, rootDir, logger) + } + + logger.Debugf("Symlink target is a directory, walking it: %s", target) + // Recursively walk the target directory, as it will not be included in this walk + return filepath.WalkDir(target, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("walk target dir: %w", err) + } - if d.IsDir() { + inf, err := d.Info() + if err != nil { + return fmt.Errorf("get file info: %w", err) + } + + return collectFileMetadata(path, inf, inf.IsDir(), fgi, false, followSymlinks, rootDir, logger) + }) +} + +func collectFileMetadata(path string, fileInfo fs.FileInfo, isDirectory bool, fgi *fileGroupInfoCollector, collectAttributes, followSymlinks bool, rootDir string, logger log.Logger) error { + absPath, err := filepath.Abs(path) + if err != nil { + return fmt.Errorf("get absolute path: %w", err) + } + if isDirectory { fgi.AddDir(&DirectoryInfo{ Path: absPath, - ModTime: inf.ModTime(), + ModTime: fileInfo.ModTime(), }) return nil } - if inf.Mode()&os.ModeSymlink != 0 { - logger.Debugf("Skipping symbolic link: %s", path) - - return nil + if fileInfo.Mode()&os.ModeSymlink != 0 { + return collectSymlink(path, fgi, followSymlinks, rootDir, logger) } file, err := os.Open(path) @@ -159,10 +197,10 @@ func collectFileMetadata(path string, d fs.DirEntry, fgi *fileGroupInfoCollector fgi.AddFile(&FileInfo{ Path: savedPath, - Size: inf.Size(), + Size: fileInfo.Size(), Hash: hash, - ModTime: inf.ModTime(), - Mode: inf.Mode(), + ModTime: fileInfo.ModTime(), + Mode: fileInfo.Mode(), Attributes: attrs, }) @@ -265,3 +303,35 @@ func restoreDirectoryInfo(dir DirectoryInfo, rootDir string) error { return nil } + +func resolveSymlink(path string) (string, error) { + seen := make(map[string]struct{}) + current := path + + for { + if _, visited := seen[current]; visited { + return "", fmt.Errorf("circular symlink detected at %s", current) + } + seen[current] = struct{}{} + + target, err := os.Readlink(current) + if err != nil { + return "", fmt.Errorf("read symlink: %w", err) + } + + if !filepath.IsAbs(target) { + target = filepath.Join(filepath.Dir(current), target) + } + + info, err := os.Lstat(target) + if err != nil { + return "", fmt.Errorf("lstat target: %w", err) + } + + if info.Mode()&os.ModeSymlink == 0 { + return target, nil + } + + current = target + } +} diff --git a/internal/xcode/metadata.go b/internal/xcode/metadata.go index 3cae9b85..a99d4908 100644 --- a/internal/xcode/metadata.go +++ b/internal/xcode/metadata.go @@ -32,6 +32,7 @@ type CreateMetadataParams struct { DerivedDataPath string XcodeCacheDirPath string CacheKey string + FollowSymlinks bool } func CreateMetadata(params CreateMetadataParams, envProvider func(string) string, logger log.Logger) (*Metadata, error) { @@ -39,14 +40,22 @@ func CreateMetadata(params CreateMetadataParams, envProvider func(string) string return nil, fmt.Errorf("missing project root directory path") } var projectFiles FileGroupInfo - projectFiles, err := collectFileGroupInfo(params.ProjectRootDirPath, params.ProjectRootDirPath, true, logger) + projectFiles, err := collectFileGroupInfo(params.ProjectRootDirPath, + params.ProjectRootDirPath, + true, + params.FollowSymlinks, + logger) if err != nil { return nil, fmt.Errorf("calculate project files info: %w", err) } var derivedData FileGroupInfo if params.DerivedDataPath != "" { - derivedData, err = collectFileGroupInfo(params.DerivedDataPath, "", false, logger) + derivedData, err = collectFileGroupInfo(params.DerivedDataPath, + "", + false, + params.FollowSymlinks, + logger) if err != nil { return nil, fmt.Errorf("calculate derived data info: %w", err) } @@ -54,7 +63,11 @@ func CreateMetadata(params CreateMetadataParams, envProvider func(string) string var xcodeCacheDir FileGroupInfo if params.XcodeCacheDirPath != "" { - xcodeCacheDir, err = collectFileGroupInfo(params.XcodeCacheDirPath, "", false, logger) + xcodeCacheDir, err = collectFileGroupInfo(params.XcodeCacheDirPath, + "", + false, + params.FollowSymlinks, + logger) if err != nil { return nil, fmt.Errorf("calculate xcode cache dir info: %w", err) } From 1d8f62cdcc482214370515e09a0a481bbbf21350 Mon Sep 17 00:00:00 2001 From: zsolt-marta-bitrise Date: Tue, 10 Sep 2024 15:41:03 +0200 Subject: [PATCH 02/10] feat: ACI-2805 Save symlink metadata --- cmd/restoreXcodeDerivedDataFiles.go | 24 ++++ internal/xcode/file_group_info.go | 168 +++++++++++++++++----------- internal/xcode/metadata.go | 19 +++- 3 files changed, 143 insertions(+), 68 deletions(-) diff --git a/cmd/restoreXcodeDerivedDataFiles.go b/cmd/restoreXcodeDerivedDataFiles.go index 468a6332..7e9dc3b7 100644 --- a/cmd/restoreXcodeDerivedDataFiles.go +++ b/cmd/restoreXcodeDerivedDataFiles.go @@ -154,6 +154,20 @@ func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Ca return op, fmt.Errorf("restore DerivedData directories: %w", err) } + if len(metadata.ProjectFiles.Symlinks) > 0 { + logger.TInfof("Restoring project symlinks") + if filesUpdated, err = xcode.RestoreSymlinks(metadata.ProjectFiles.Symlinks, logger); err != nil { + return op, fmt.Errorf("restore project symlink: %w", err) + } + } + + if len(metadata.DerivedData.Symlinks) > 0 { + logger.TInfof("Restoring DerivedData symlinks") + if filesUpdated, err = xcode.RestoreSymlinks(metadata.DerivedData.Symlinks, logger); err != nil { + return op, fmt.Errorf("restore DerivedData symlink: %w", err) + } + } + if len(metadata.XcodeCacheDir.Files) > 0 { logger.TInfof("Downloading Xcode cache files") if _, err := xcode.DownloadCacheFilesFromBuildCache(ctx, metadata.XcodeCacheDir, kvClient, logger, isDebugLogMode, forceOverwrite, maxLoggedDownloadErrors); err != nil { @@ -166,6 +180,13 @@ func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Ca } } + if len(metadata.XcodeCacheDir.Symlinks) > 0 { + logger.TInfof("Restoring Xcode cache symlinks") + if filesUpdated, err = xcode.RestoreSymlinks(metadata.XcodeCacheDir.Symlinks, logger); err != nil { + return op, fmt.Errorf("restore xcode symlink: %w", err) + } + } + return op, nil } @@ -234,8 +255,11 @@ func logCacheMetadata(md *xcode.Metadata, logger log.Logger, isDebugLogMode bool logger.Infof(" Git commit: %s", md.GitCommit) logger.Infof(" Git branch: %s", md.GitBranch) logger.Infof(" Project files: %d", len(md.ProjectFiles.Files)) + logger.Infof(" Project sylinks: %d", len(md.ProjectFiles.Symlinks)) logger.Infof(" DerivedData files: %d", len(md.DerivedData.Files)) + logger.Infof(" DerivedData symlinks: %d", len(md.DerivedData.Symlinks)) logger.Infof(" Xcode cache files: %d", len(md.XcodeCacheDir.Files)) + logger.Infof(" Xcode cache symlinks: %d", len(md.XcodeCacheDir.Symlinks)) logger.Infof(" Build Cache CLI version: %s", md.BuildCacheCLIVersion) logger.Infof(" Metadata version: %d", md.MetadataVersion) diff --git a/internal/xcode/file_group_info.go b/internal/xcode/file_group_info.go index 1aae7844..e70dc728 100644 --- a/internal/xcode/file_group_info.go +++ b/internal/xcode/file_group_info.go @@ -1,10 +1,7 @@ package xcode import ( - "crypto/sha256" - "encoding/hex" "fmt" - "io" "io/fs" "os" "path/filepath" @@ -14,12 +11,18 @@ import ( "github.com/dustin/go-humanize" "github.com/pkg/xattr" + "crypto/sha256" + "encoding/hex" + "io" + "syscall" + "github.com/bitrise-io/go-utils/v2/log" ) type FileGroupInfo struct { Files []*FileInfo `json:"files"` Directories []*DirectoryInfo `json:"directories"` + Symlinks []*SymlinkInfo `json:"symlinks,omitempty"` } type DirectoryInfo struct { @@ -27,6 +30,12 @@ type DirectoryInfo struct { ModTime time.Time `json:"modTime"` } +type SymlinkInfo struct { + Path string `json:"path"` + Target string `json:"target"` + ModTime time.Time `json:"modTime"` +} + type FileInfo struct { Path string `json:"path"` Size int64 `json:"size"` @@ -39,8 +48,11 @@ type FileInfo struct { type fileGroupInfoCollector struct { Files []*FileInfo Dirs []*DirectoryInfo + Symlinks []*SymlinkInfo LargestFileSize int64 - mu sync.Mutex + seen map[string]bool + + mu sync.Mutex } func (mc *fileGroupInfoCollector) AddFile(fileInfo *FileInfo) { @@ -50,20 +62,38 @@ func (mc *fileGroupInfoCollector) AddFile(fileInfo *FileInfo) { if fileInfo.Size > mc.LargestFileSize { mc.LargestFileSize = fileInfo.Size } + mc.seen[fileInfo.Path] = true } func (mc *fileGroupInfoCollector) AddDir(dirInfo *DirectoryInfo) { mc.mu.Lock() defer mc.mu.Unlock() mc.Dirs = append(mc.Dirs, dirInfo) + mc.seen[dirInfo.Path] = true } -func collectFileGroupInfo(cacheDirPath string, rootDir string, collectAttributes, followSymlinks bool, logger log.Logger) (FileGroupInfo, error) { +func (mc *fileGroupInfoCollector) AddSymlink(symlink *SymlinkInfo) { + mc.mu.Lock() + defer mc.mu.Unlock() + mc.Symlinks = append(mc.Symlinks, symlink) + mc.seen[symlink.Path] = true +} + +func (mc *fileGroupInfoCollector) isSeen(path string) bool { + mc.mu.Lock() + defer mc.mu.Unlock() + + return mc.seen[path] +} + +func collectFileGroupInfo(cacheDirPath string, collectAttributes, followSymlinks bool, logger log.Logger) (FileGroupInfo, error) { var dd FileGroupInfo fgi := fileGroupInfoCollector{ - Files: make([]*FileInfo, 0), - Dirs: make([]*DirectoryInfo, 0), + Files: make([]*FileInfo, 0), + Dirs: make([]*DirectoryInfo, 0), + Symlinks: make([]*SymlinkInfo, 0), + seen: make(map[string]bool), } var wg sync.WaitGroup semaphore := make(chan struct{}, 10) // Limit parallelization @@ -87,7 +117,10 @@ func collectFileGroupInfo(cacheDirPath string, rootDir string, collectAttributes return } - if err := collectFileMetadata(path, inf, inf.IsDir(), &fgi, collectAttributes, followSymlinks, rootDir, logger); err != nil { + if !filepath.IsAbs(path) { + path = filepath.Join(cacheDirPath, path) + } + if err := collectFileMetadata(path, inf, inf.IsDir(), &fgi, collectAttributes, followSymlinks, logger); err != nil { logger.Errorf("Failed to collect metadata: %s", err) } }(d) @@ -111,25 +144,34 @@ func collectFileGroupInfo(cacheDirPath string, rootDir string, collectAttributes } // nolint:wrapcheck -func collectSymlink(path string, fgi *fileGroupInfoCollector, followSymlinks bool, rootDir string, logger log.Logger) error { +func followSymlink(path string, target string, fgi *fileGroupInfoCollector, followSymlinks bool, logger log.Logger) error { if !followSymlinks { logger.Debugf("Skipping symbolic link: %s", path) return nil } + if fgi.isSeen(target) { + logger.Debugf("Skipping symbolic link target: %s, already seen", target) - target, err := resolveSymlink(path) - if err != nil { - return fmt.Errorf("resolve symlink %s: %w", path, err) + return nil } - logger.Debugf("Resolved symlink %s to %s", path, target) + // Dont save symlink if target doesn't exist stat, err := os.Stat(target) if err != nil { return fmt.Errorf("stat target: %w", err) } + + logger.Debugf("Resolved symlink %s to target: %s", path, target) + + fgi.AddSymlink(&SymlinkInfo{ + Path: path, + Target: target, + ModTime: stat.ModTime(), + }) + if !stat.IsDir() { - return collectFileMetadata(target, stat, false, fgi, false, followSymlinks, rootDir, logger) + return collectFileMetadata(target, stat, false, fgi, false, followSymlinks, logger) } logger.Debugf("Symlink target is a directory, walking it: %s", target) @@ -144,26 +186,43 @@ func collectSymlink(path string, fgi *fileGroupInfoCollector, followSymlinks boo return fmt.Errorf("get file info: %w", err) } - return collectFileMetadata(path, inf, inf.IsDir(), fgi, false, followSymlinks, rootDir, logger) + if !filepath.IsAbs(path) { + path = filepath.Join(target, path) + } + + return collectFileMetadata(path, inf, inf.IsDir(), fgi, false, followSymlinks, logger) }) } -func collectFileMetadata(path string, fileInfo fs.FileInfo, isDirectory bool, fgi *fileGroupInfoCollector, collectAttributes, followSymlinks bool, rootDir string, logger log.Logger) error { - absPath, err := filepath.Abs(path) - if err != nil { - return fmt.Errorf("get absolute path: %w", err) +func collectFileMetadata(path string, fileInfo fs.FileInfo, isDirectory bool, fgi *fileGroupInfoCollector, collectAttributes, followSymlinks bool, logger log.Logger) error { + if fgi.isSeen(path) { + logger.Debugf("Skipping path %s, already seen", path) + + return nil } + if isDirectory { fgi.AddDir(&DirectoryInfo{ - Path: absPath, + Path: path, ModTime: fileInfo.ModTime(), }) return nil } - if fileInfo.Mode()&os.ModeSymlink != 0 { - return collectSymlink(path, fgi, followSymlinks, rootDir, logger) + isSymlink := fileInfo.Mode()&os.ModeSymlink != 0 + + if isSymlink { + var target string + target, err := os.Readlink(path) + if err != nil { + return fmt.Errorf("read symlink: %w", err) + } + if !filepath.IsAbs(target) { + target = filepath.Join(filepath.Dir(path), target) + } + + return followSymlink(path, target, fgi, followSymlinks, logger) } file, err := os.Open(path) @@ -178,15 +237,6 @@ func collectFileMetadata(path string, fileInfo fs.FileInfo, isDirectory bool, fg } hash := hex.EncodeToString(hasher.Sum(nil)) - savedPath := absPath - if rootDir != "" { - relPath, err := filepath.Rel(rootDir, path) - if err != nil { - return fmt.Errorf("get relative path: %w", err) - } - savedPath = relPath - } - var attrs map[string]string if collectAttributes { attrs, err = getAttributes(path) @@ -196,7 +246,7 @@ func collectFileMetadata(path string, fileInfo fs.FileInfo, isDirectory bool, fg } fgi.AddFile(&FileInfo{ - Path: savedPath, + Path: path, Size: fileInfo.Size(), Hash: hash, ModTime: fileInfo.ModTime(), @@ -235,6 +285,26 @@ func setAttributes(path string, attributes map[string]string) error { return nil } +func restoreSymlink(symlink SymlinkInfo, logger log.Logger) bool { + err := os.Symlink(symlink.Target, symlink.Path) + if err != nil { + logger.Infof("Error creating symlink %s -> %s: %v", symlink.Path, symlink.Target, err) + + return false + } + + // Set times + mtimeSpec := syscall.NsecToTimespec(symlink.ModTime.UnixNano()) + err = syscall.UtimesNano(symlink.Path, []syscall.Timespec{mtimeSpec, mtimeSpec}) + if err != nil { + logger.Debugf("Error setting symlink times for %s: %v", symlink.Path, err) + + return false + } + + return true +} + func restoreFileInfo(fi FileInfo, rootDir string, logger log.Logger) bool { var path string if filepath.IsAbs(fi.Path) { @@ -268,7 +338,7 @@ func restoreFileInfo(fi FileInfo, rootDir string, logger log.Logger) bool { } if err = os.Chmod(fi.Path, fi.Mode); err != nil { - logger.Debugf("Error setting file mode time for %s: %v", fi.Path, err) + logger.Debugf("Error setting file mode for %s: %v", fi.Path, err) return false } @@ -303,35 +373,3 @@ func restoreDirectoryInfo(dir DirectoryInfo, rootDir string) error { return nil } - -func resolveSymlink(path string) (string, error) { - seen := make(map[string]struct{}) - current := path - - for { - if _, visited := seen[current]; visited { - return "", fmt.Errorf("circular symlink detected at %s", current) - } - seen[current] = struct{}{} - - target, err := os.Readlink(current) - if err != nil { - return "", fmt.Errorf("read symlink: %w", err) - } - - if !filepath.IsAbs(target) { - target = filepath.Join(filepath.Dir(current), target) - } - - info, err := os.Lstat(target) - if err != nil { - return "", fmt.Errorf("lstat target: %w", err) - } - - if info.Mode()&os.ModeSymlink == 0 { - return target, nil - } - - current = target - } -} diff --git a/internal/xcode/metadata.go b/internal/xcode/metadata.go index a99d4908..3dfa2e35 100644 --- a/internal/xcode/metadata.go +++ b/internal/xcode/metadata.go @@ -41,7 +41,6 @@ func CreateMetadata(params CreateMetadataParams, envProvider func(string) string } var projectFiles FileGroupInfo projectFiles, err := collectFileGroupInfo(params.ProjectRootDirPath, - params.ProjectRootDirPath, true, params.FollowSymlinks, logger) @@ -52,7 +51,6 @@ func CreateMetadata(params CreateMetadataParams, envProvider func(string) string var derivedData FileGroupInfo if params.DerivedDataPath != "" { derivedData, err = collectFileGroupInfo(params.DerivedDataPath, - "", false, params.FollowSymlinks, logger) @@ -64,7 +62,6 @@ func CreateMetadata(params CreateMetadataParams, envProvider func(string) string var xcodeCacheDir FileGroupInfo if params.XcodeCacheDirPath != "" { xcodeCacheDir, err = collectFileGroupInfo(params.XcodeCacheDirPath, - "", false, params.FollowSymlinks, logger) @@ -146,6 +143,22 @@ func RestoreDirectoryInfos(dirInfos []*DirectoryInfo, rootDir string, logger log return nil } +func RestoreSymlinks(symlinks []*SymlinkInfo, logger log.Logger) (int, error) { + updated := 0 + + logger.Infof("(i) %d symlinks' info loaded from cache metadata", len(symlinks)) + + for _, si := range symlinks { + if restoreSymlink(*si, logger) { + updated++ + } + } + + logger.Infof("(i) %d symlinks restored", updated) + + return updated, nil +} + func RestoreFileInfos(fileInfos []*FileInfo, rootDir string, logger log.Logger) (int, error) { updated := 0 From b77ec22334d8904fffd495e1f40c432c5dbb3cde Mon Sep 17 00:00:00 2001 From: zsolt-marta-bitrise Date: Tue, 10 Sep 2024 15:46:37 +0200 Subject: [PATCH 03/10] feat: ACI-2805 Lintfix --- cmd/restoreXcodeDerivedDataFiles.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/restoreXcodeDerivedDataFiles.go b/cmd/restoreXcodeDerivedDataFiles.go index 7e9dc3b7..a349fc1d 100644 --- a/cmd/restoreXcodeDerivedDataFiles.go +++ b/cmd/restoreXcodeDerivedDataFiles.go @@ -156,14 +156,14 @@ func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Ca if len(metadata.ProjectFiles.Symlinks) > 0 { logger.TInfof("Restoring project symlinks") - if filesUpdated, err = xcode.RestoreSymlinks(metadata.ProjectFiles.Symlinks, logger); err != nil { + if _, err = xcode.RestoreSymlinks(metadata.ProjectFiles.Symlinks, logger); err != nil { return op, fmt.Errorf("restore project symlink: %w", err) } } if len(metadata.DerivedData.Symlinks) > 0 { logger.TInfof("Restoring DerivedData symlinks") - if filesUpdated, err = xcode.RestoreSymlinks(metadata.DerivedData.Symlinks, logger); err != nil { + if _, err = xcode.RestoreSymlinks(metadata.DerivedData.Symlinks, logger); err != nil { return op, fmt.Errorf("restore DerivedData symlink: %w", err) } } @@ -182,7 +182,7 @@ func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Ca if len(metadata.XcodeCacheDir.Symlinks) > 0 { logger.TInfof("Restoring Xcode cache symlinks") - if filesUpdated, err = xcode.RestoreSymlinks(metadata.XcodeCacheDir.Symlinks, logger); err != nil { + if _, err = xcode.RestoreSymlinks(metadata.XcodeCacheDir.Symlinks, logger); err != nil { return op, fmt.Errorf("restore xcode symlink: %w", err) } } From 1a365842fd6206f041b4ae3cce553ec8be93e69b Mon Sep 17 00:00:00 2001 From: zsolt-marta-bitrise Date: Tue, 10 Sep 2024 17:15:54 +0200 Subject: [PATCH 04/10] fix: ACI-2805 Test for absolute path --- internal/xcode/metadata_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/xcode/metadata_test.go b/internal/xcode/metadata_test.go index 48da711d..cc45ff16 100644 --- a/internal/xcode/metadata_test.go +++ b/internal/xcode/metadata_test.go @@ -66,7 +66,7 @@ func Test_CreateMetadata(t *testing.T) { require.Len(t, md.ProjectFiles.Files, 1) fi := md.ProjectFiles.Files[0] - require.True(t, strings.HasPrefix(fi.Path, "test-file.swift")) + require.True(t, strings.Contains(fi.Path, "test-file.swift")) require.NotEmpty(t, fi.Hash) require.NotEmpty(t, md.CacheKey) From a35f96e3974b19fa12ad5e5f96329f1e1400ac4d Mon Sep 17 00:00:00 2001 From: zsolt-marta-bitrise Date: Tue, 10 Sep 2024 23:21:13 +0200 Subject: [PATCH 05/10] fix: ACI-2805 Save symlinks from FGI --- internal/xcode/file_group_info.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/xcode/file_group_info.go b/internal/xcode/file_group_info.go index e70dc728..e5750ae1 100644 --- a/internal/xcode/file_group_info.go +++ b/internal/xcode/file_group_info.go @@ -136,6 +136,7 @@ func collectFileGroupInfo(cacheDirPath string, collectAttributes, followSymlinks dd.Files = fgi.Files dd.Directories = fgi.Dirs + dd.Symlinks = fgi.Symlinks logger.Infof("(i) Collected %d files and %d directories ", len(dd.Files), len(dd.Directories)) logger.Debugf("(i) Largest processed file size: %s", humanize.Bytes(uint64(fgi.LargestFileSize))) From 5b2402c5410be9fc9419c5d382712bc9a1bdb356 Mon Sep 17 00:00:00 2001 From: zsolt-marta-bitrise Date: Wed, 11 Sep 2024 09:30:10 +0200 Subject: [PATCH 06/10] fix: ACI-2805 Skip existing symlinks --- internal/xcode/file_group_info.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/internal/xcode/file_group_info.go b/internal/xcode/file_group_info.go index e5750ae1..c896bb75 100644 --- a/internal/xcode/file_group_info.go +++ b/internal/xcode/file_group_info.go @@ -287,9 +287,16 @@ func setAttributes(path string, attributes map[string]string) error { } func restoreSymlink(symlink SymlinkInfo, logger log.Logger) bool { - err := os.Symlink(symlink.Target, symlink.Path) + fileInfo, err := os.Stat(symlink.Path) + if err == nil && fileInfo.Mode()&os.ModeSymlink != 0 { + logger.Debugf("Symlink %s already exists", symlink.Path) + + return false + } + + err = os.Symlink(symlink.Target, symlink.Path) if err != nil { - logger.Infof("Error creating symlink %s -> %s: %v", symlink.Path, symlink.Target, err) + logger.Debugf("Error creating symlink %s -> %s: %v", symlink.Path, symlink.Target, err) return false } From 31d21d69fe469035779364553362101ed167fff7 Mon Sep 17 00:00:00 2001 From: zsolt-marta-bitrise Date: Wed, 11 Sep 2024 09:42:24 +0200 Subject: [PATCH 07/10] fix: ACI-2805 Skip existing symlinks --- cmd/restoreXcodeDerivedDataFiles.go | 6 +++--- internal/xcode/file_group_info.go | 13 ++++++++++--- internal/xcode/metadata.go | 4 ++-- 3 files changed, 15 insertions(+), 8 deletions(-) diff --git a/cmd/restoreXcodeDerivedDataFiles.go b/cmd/restoreXcodeDerivedDataFiles.go index a349fc1d..3dff2a09 100644 --- a/cmd/restoreXcodeDerivedDataFiles.go +++ b/cmd/restoreXcodeDerivedDataFiles.go @@ -156,14 +156,14 @@ func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Ca if len(metadata.ProjectFiles.Symlinks) > 0 { logger.TInfof("Restoring project symlinks") - if _, err = xcode.RestoreSymlinks(metadata.ProjectFiles.Symlinks, logger); err != nil { + if _, err = xcode.RestoreSymlinks(metadata.ProjectFiles.Symlinks, forceOverwrite, logger); err != nil { return op, fmt.Errorf("restore project symlink: %w", err) } } if len(metadata.DerivedData.Symlinks) > 0 { logger.TInfof("Restoring DerivedData symlinks") - if _, err = xcode.RestoreSymlinks(metadata.DerivedData.Symlinks, logger); err != nil { + if _, err = xcode.RestoreSymlinks(metadata.DerivedData.Symlinks, forceOverwrite, logger); err != nil { return op, fmt.Errorf("restore DerivedData symlink: %w", err) } } @@ -182,7 +182,7 @@ func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Ca if len(metadata.XcodeCacheDir.Symlinks) > 0 { logger.TInfof("Restoring Xcode cache symlinks") - if _, err = xcode.RestoreSymlinks(metadata.XcodeCacheDir.Symlinks, logger); err != nil { + if _, err = xcode.RestoreSymlinks(metadata.XcodeCacheDir.Symlinks, forceOverwrite, logger); err != nil { return op, fmt.Errorf("restore xcode symlink: %w", err) } } diff --git a/internal/xcode/file_group_info.go b/internal/xcode/file_group_info.go index c896bb75..452e4b8a 100644 --- a/internal/xcode/file_group_info.go +++ b/internal/xcode/file_group_info.go @@ -286,12 +286,19 @@ func setAttributes(path string, attributes map[string]string) error { return nil } -func restoreSymlink(symlink SymlinkInfo, logger log.Logger) bool { +func restoreSymlink(symlink SymlinkInfo, forceOverwrite bool, logger log.Logger) bool { fileInfo, err := os.Stat(symlink.Path) if err == nil && fileInfo.Mode()&os.ModeSymlink != 0 { - logger.Debugf("Symlink %s already exists", symlink.Path) + if !forceOverwrite { + logger.Debugf("Symlink %s already exists", symlink.Path) + return false + } - return false + err := os.Remove(symlink.Path) + if err != nil { + logger.Infof("Error removing existing symlink %s: %v", symlink.Path, err) + return false + } } err = os.Symlink(symlink.Target, symlink.Path) diff --git a/internal/xcode/metadata.go b/internal/xcode/metadata.go index 3dfa2e35..d540d9a9 100644 --- a/internal/xcode/metadata.go +++ b/internal/xcode/metadata.go @@ -143,13 +143,13 @@ func RestoreDirectoryInfos(dirInfos []*DirectoryInfo, rootDir string, logger log return nil } -func RestoreSymlinks(symlinks []*SymlinkInfo, logger log.Logger) (int, error) { +func RestoreSymlinks(symlinks []*SymlinkInfo, forceOverwrite bool, logger log.Logger) (int, error) { updated := 0 logger.Infof("(i) %d symlinks' info loaded from cache metadata", len(symlinks)) for _, si := range symlinks { - if restoreSymlink(*si, logger) { + if restoreSymlink(*si, forceOverwrite, logger) { updated++ } } From 9acf2ced9db49eb267b8c243fa4c815c0744dc1f Mon Sep 17 00:00:00 2001 From: zsolt-marta-bitrise Date: Wed, 11 Sep 2024 09:43:10 +0200 Subject: [PATCH 08/10] fix: ACI-2805 Lintfix --- internal/xcode/file_group_info.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/internal/xcode/file_group_info.go b/internal/xcode/file_group_info.go index 452e4b8a..455c40d5 100644 --- a/internal/xcode/file_group_info.go +++ b/internal/xcode/file_group_info.go @@ -291,12 +291,14 @@ func restoreSymlink(symlink SymlinkInfo, forceOverwrite bool, logger log.Logger) if err == nil && fileInfo.Mode()&os.ModeSymlink != 0 { if !forceOverwrite { logger.Debugf("Symlink %s already exists", symlink.Path) + return false } err := os.Remove(symlink.Path) if err != nil { logger.Infof("Error removing existing symlink %s: %v", symlink.Path, err) + return false } } From f248fb412b2b77d02f72daec228f1202fe20b61c Mon Sep 17 00:00:00 2001 From: zsolt-marta-bitrise Date: Wed, 11 Sep 2024 09:49:38 +0200 Subject: [PATCH 09/10] fix: ACI-2805 Use Lstat --- internal/xcode/file_group_info.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/xcode/file_group_info.go b/internal/xcode/file_group_info.go index 455c40d5..7982fb30 100644 --- a/internal/xcode/file_group_info.go +++ b/internal/xcode/file_group_info.go @@ -287,7 +287,7 @@ func setAttributes(path string, attributes map[string]string) error { } func restoreSymlink(symlink SymlinkInfo, forceOverwrite bool, logger log.Logger) bool { - fileInfo, err := os.Stat(symlink.Path) + fileInfo, err := os.Lstat(symlink.Path) if err == nil && fileInfo.Mode()&os.ModeSymlink != 0 { if !forceOverwrite { logger.Debugf("Symlink %s already exists", symlink.Path) From c1dbce1df22882b12be48fc5ecfac2b07af0cebb Mon Sep 17 00:00:00 2001 From: zsolt-marta-bitrise Date: Wed, 11 Sep 2024 11:53:20 +0200 Subject: [PATCH 10/10] fix: ACI-2805 Always overwrite symlinks --- cmd/restoreXcodeDerivedDataFiles.go | 8 ++++---- cmd/saveXcodeDerivedDataFiles.go | 2 +- internal/xcode/file_group_info.go | 8 ++------ internal/xcode/metadata.go | 4 ++-- 4 files changed, 9 insertions(+), 13 deletions(-) diff --git a/cmd/restoreXcodeDerivedDataFiles.go b/cmd/restoreXcodeDerivedDataFiles.go index 3dff2a09..ee4f2698 100644 --- a/cmd/restoreXcodeDerivedDataFiles.go +++ b/cmd/restoreXcodeDerivedDataFiles.go @@ -156,14 +156,14 @@ func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Ca if len(metadata.ProjectFiles.Symlinks) > 0 { logger.TInfof("Restoring project symlinks") - if _, err = xcode.RestoreSymlinks(metadata.ProjectFiles.Symlinks, forceOverwrite, logger); err != nil { + if _, err = xcode.RestoreSymlinks(metadata.ProjectFiles.Symlinks, logger); err != nil { return op, fmt.Errorf("restore project symlink: %w", err) } } if len(metadata.DerivedData.Symlinks) > 0 { logger.TInfof("Restoring DerivedData symlinks") - if _, err = xcode.RestoreSymlinks(metadata.DerivedData.Symlinks, forceOverwrite, logger); err != nil { + if _, err = xcode.RestoreSymlinks(metadata.DerivedData.Symlinks, logger); err != nil { return op, fmt.Errorf("restore DerivedData symlink: %w", err) } } @@ -182,7 +182,7 @@ func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Ca if len(metadata.XcodeCacheDir.Symlinks) > 0 { logger.TInfof("Restoring Xcode cache symlinks") - if _, err = xcode.RestoreSymlinks(metadata.XcodeCacheDir.Symlinks, forceOverwrite, logger); err != nil { + if _, err = xcode.RestoreSymlinks(metadata.XcodeCacheDir.Symlinks, logger); err != nil { return op, fmt.Errorf("restore xcode symlink: %w", err) } } @@ -255,7 +255,7 @@ func logCacheMetadata(md *xcode.Metadata, logger log.Logger, isDebugLogMode bool logger.Infof(" Git commit: %s", md.GitCommit) logger.Infof(" Git branch: %s", md.GitBranch) logger.Infof(" Project files: %d", len(md.ProjectFiles.Files)) - logger.Infof(" Project sylinks: %d", len(md.ProjectFiles.Symlinks)) + logger.Infof(" Project symlinks: %d", len(md.ProjectFiles.Symlinks)) logger.Infof(" DerivedData files: %d", len(md.DerivedData.Files)) logger.Infof(" DerivedData symlinks: %d", len(md.DerivedData.Symlinks)) logger.Infof(" Xcode cache files: %d", len(md.XcodeCacheDir.Files)) diff --git a/cmd/saveXcodeDerivedDataFiles.go b/cmd/saveXcodeDerivedDataFiles.go index 4591b968..ab2679ce 100644 --- a/cmd/saveXcodeDerivedDataFiles.go +++ b/cmd/saveXcodeDerivedDataFiles.go @@ -98,7 +98,7 @@ func init() { panic(err) } saveXcodeDerivedDataFilesCmd.Flags().String("xcodecache-path", "", "Path to the Xcode cache directory folder to be saved. If not set, it will not be uploaded.") - saveXcodeDerivedDataFilesCmd.Flags().Bool("follow-symlinks", false, "Follow symlinks when calculating metadata (default: false)") + saveXcodeDerivedDataFilesCmd.Flags().Bool("follow-symlinks", false, "Follow symlinks when calculating metadata and save referenced files to the cache (default: false)") } func saveXcodeDerivedDataFilesCmdFn(ctx context.Context, diff --git a/internal/xcode/file_group_info.go b/internal/xcode/file_group_info.go index 7982fb30..e8f65518 100644 --- a/internal/xcode/file_group_info.go +++ b/internal/xcode/file_group_info.go @@ -286,14 +286,10 @@ func setAttributes(path string, attributes map[string]string) error { return nil } -func restoreSymlink(symlink SymlinkInfo, forceOverwrite bool, logger log.Logger) bool { +func restoreSymlink(symlink SymlinkInfo, logger log.Logger) bool { fileInfo, err := os.Lstat(symlink.Path) if err == nil && fileInfo.Mode()&os.ModeSymlink != 0 { - if !forceOverwrite { - logger.Debugf("Symlink %s already exists", symlink.Path) - - return false - } + logger.Debugf("Symlink %s already exists, overwriting...", symlink.Path) err := os.Remove(symlink.Path) if err != nil { diff --git a/internal/xcode/metadata.go b/internal/xcode/metadata.go index d540d9a9..3dfa2e35 100644 --- a/internal/xcode/metadata.go +++ b/internal/xcode/metadata.go @@ -143,13 +143,13 @@ func RestoreDirectoryInfos(dirInfos []*DirectoryInfo, rootDir string, logger log return nil } -func RestoreSymlinks(symlinks []*SymlinkInfo, forceOverwrite bool, logger log.Logger) (int, error) { +func RestoreSymlinks(symlinks []*SymlinkInfo, logger log.Logger) (int, error) { updated := 0 logger.Infof("(i) %d symlinks' info loaded from cache metadata", len(symlinks)) for _, si := range symlinks { - if restoreSymlink(*si, forceOverwrite, logger) { + if restoreSymlink(*si, logger) { updated++ } }