diff --git a/cmd/restoreXcodeDerivedDataFiles.go b/cmd/restoreXcodeDerivedDataFiles.go index 468a6332..ee4f2698 100644 --- a/cmd/restoreXcodeDerivedDataFiles.go +++ b/cmd/restoreXcodeDerivedDataFiles.go @@ -154,6 +154,20 @@ func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Ca return op, fmt.Errorf("restore DerivedData directories: %w", err) } + if len(metadata.ProjectFiles.Symlinks) > 0 { + logger.TInfof("Restoring project symlinks") + if _, err = xcode.RestoreSymlinks(metadata.ProjectFiles.Symlinks, logger); err != nil { + return op, fmt.Errorf("restore project symlink: %w", err) + } + } + + if len(metadata.DerivedData.Symlinks) > 0 { + logger.TInfof("Restoring DerivedData symlinks") + if _, err = xcode.RestoreSymlinks(metadata.DerivedData.Symlinks, logger); err != nil { + return op, fmt.Errorf("restore DerivedData symlink: %w", err) + } + } + if len(metadata.XcodeCacheDir.Files) > 0 { logger.TInfof("Downloading Xcode cache files") if _, err := xcode.DownloadCacheFilesFromBuildCache(ctx, metadata.XcodeCacheDir, kvClient, logger, isDebugLogMode, forceOverwrite, maxLoggedDownloadErrors); err != nil { @@ -166,6 +180,13 @@ func restoreXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Ca } } + if len(metadata.XcodeCacheDir.Symlinks) > 0 { + logger.TInfof("Restoring Xcode cache symlinks") + if _, err = xcode.RestoreSymlinks(metadata.XcodeCacheDir.Symlinks, logger); err != nil { + return op, fmt.Errorf("restore xcode symlink: %w", err) + } + } + return op, nil } @@ -234,8 +255,11 @@ func logCacheMetadata(md *xcode.Metadata, logger log.Logger, isDebugLogMode bool logger.Infof(" Git commit: %s", md.GitCommit) logger.Infof(" Git branch: %s", md.GitBranch) logger.Infof(" Project files: %d", len(md.ProjectFiles.Files)) + logger.Infof(" Project symlinks: %d", len(md.ProjectFiles.Symlinks)) logger.Infof(" DerivedData files: %d", len(md.DerivedData.Files)) + logger.Infof(" DerivedData symlinks: %d", len(md.DerivedData.Symlinks)) logger.Infof(" Xcode cache files: %d", len(md.XcodeCacheDir.Files)) + logger.Infof(" Xcode cache symlinks: %d", len(md.XcodeCacheDir.Symlinks)) logger.Infof(" Build Cache CLI version: %s", md.BuildCacheCLIVersion) logger.Infof(" Metadata version: %d", md.MetadataVersion) diff --git a/cmd/saveXcodeDerivedDataFiles.go b/cmd/saveXcodeDerivedDataFiles.go index 706b5738..ab2679ce 100644 --- a/cmd/saveXcodeDerivedDataFiles.go +++ b/cmd/saveXcodeDerivedDataFiles.go @@ -38,6 +38,7 @@ var saveXcodeDerivedDataFilesCmd = &cobra.Command{ cacheKey, _ := cmd.Flags().GetString("key") ddPath, _ := cmd.Flags().GetString("deriveddata-path") xcodeCachePath, _ := cmd.Flags().GetString("xcodecache-path") + followSymlinks, _ := cmd.Flags().GetBool("follow-symlinks") tracker := xcode.NewDefaultStepTracker("save-xcode-build-cache", os.Getenv, logger) defer tracker.Wait() @@ -49,7 +50,18 @@ var saveXcodeDerivedDataFilesCmd = &cobra.Command{ return fmt.Errorf("read auth config from environments: %w", err) } - op, cmdError := saveXcodeDerivedDataFilesCmdFn(cmd.Context(), authConfig, CacheMetadataPath, projectRoot, cacheKey, ddPath, xcodeCachePath, logger, tracker, startT, os.Getenv) + op, cmdError := saveXcodeDerivedDataFilesCmdFn(cmd.Context(), + authConfig, + CacheMetadataPath, + projectRoot, + cacheKey, + ddPath, + xcodeCachePath, + followSymlinks, + logger, + tracker, + startT, + os.Getenv) if op != nil { if cmdError != nil { errStr := cmdError.Error() @@ -86,10 +98,21 @@ func init() { panic(err) } saveXcodeDerivedDataFilesCmd.Flags().String("xcodecache-path", "", "Path to the Xcode cache directory folder to be saved. If not set, it will not be uploaded.") + saveXcodeDerivedDataFilesCmd.Flags().Bool("follow-symlinks", false, "Follow symlinks when calculating metadata and save referenced files to the cache (default: false)") } -func saveXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.CacheAuthConfig, cacheMetadataPath, projectRoot, providedCacheKey, derivedDataPath, xcodeCachePath string, - logger log.Logger, tracker xcode.StepAnalyticsTracker, startT time.Time, envProvider func(string) string) (*xa.CacheOperation, error) { +func saveXcodeDerivedDataFilesCmdFn(ctx context.Context, + authConfig common.CacheAuthConfig, + cacheMetadataPath, + projectRoot, + providedCacheKey, + derivedDataPath, + xcodeCachePath string, + followSymlinks bool, + logger log.Logger, + tracker xcode.StepAnalyticsTracker, + startT time.Time, + envProvider func(string) string) (*xa.CacheOperation, error) { var err error var cacheKey string if providedCacheKey == "" { @@ -126,6 +149,7 @@ func saveXcodeDerivedDataFilesCmdFn(ctx context.Context, authConfig common.Cache DerivedDataPath: derivedDataPath, XcodeCacheDirPath: xcodeCachePath, CacheKey: cacheKey, + FollowSymlinks: followSymlinks, }, envProvider, logger) if err != nil { return op, fmt.Errorf("create metadata: %w", err) diff --git a/cmd/saveXcodeDerivedDataFiles_test.go b/cmd/saveXcodeDerivedDataFiles_test.go index ae7dd8c6..365a13ac 100644 --- a/cmd/saveXcodeDerivedDataFiles_test.go +++ b/cmd/saveXcodeDerivedDataFiles_test.go @@ -36,7 +36,7 @@ func Test_saveXcodeDerivedDataFilesCmdFn(t *testing.T) { envVars := createEnvProvider(map[string]string{ "BITRISEIO_BITRISE_SERVICES_ACCESS_TOKEN": "ServiceAccessTokenValue", }) - _, err := saveXcodeDerivedDataFilesCmdFn(context.Background(), common.CacheAuthConfig{}, "", "", "", "", "", mockLogger, mockTracker, time.Now(), envVars) + _, err := saveXcodeDerivedDataFilesCmdFn(context.Background(), common.CacheAuthConfig{}, "", "", "", "", "", false, mockLogger, mockTracker, time.Now(), envVars) // then require.EqualError(t, err, "get cache key: cache key is required if BITRISE_GIT_BRANCH env var is not set") diff --git a/internal/xcode/file_group_info.go b/internal/xcode/file_group_info.go index a0293c8c..e8f65518 100644 --- a/internal/xcode/file_group_info.go +++ b/internal/xcode/file_group_info.go @@ -1,10 +1,7 @@ package xcode import ( - "crypto/sha256" - "encoding/hex" "fmt" - "io" "io/fs" "os" "path/filepath" @@ -14,12 +11,18 @@ import ( "github.com/dustin/go-humanize" "github.com/pkg/xattr" + "crypto/sha256" + "encoding/hex" + "io" + "syscall" + "github.com/bitrise-io/go-utils/v2/log" ) type FileGroupInfo struct { Files []*FileInfo `json:"files"` Directories []*DirectoryInfo `json:"directories"` + Symlinks []*SymlinkInfo `json:"symlinks,omitempty"` } type DirectoryInfo struct { @@ -27,6 +30,12 @@ type DirectoryInfo struct { ModTime time.Time `json:"modTime"` } +type SymlinkInfo struct { + Path string `json:"path"` + Target string `json:"target"` + ModTime time.Time `json:"modTime"` +} + type FileInfo struct { Path string `json:"path"` Size int64 `json:"size"` @@ -39,8 +48,11 @@ type FileInfo struct { type fileGroupInfoCollector struct { Files []*FileInfo Dirs []*DirectoryInfo + Symlinks []*SymlinkInfo LargestFileSize int64 - mu sync.Mutex + seen map[string]bool + + mu sync.Mutex } func (mc *fileGroupInfoCollector) AddFile(fileInfo *FileInfo) { @@ -50,20 +62,38 @@ func (mc *fileGroupInfoCollector) AddFile(fileInfo *FileInfo) { if fileInfo.Size > mc.LargestFileSize { mc.LargestFileSize = fileInfo.Size } + mc.seen[fileInfo.Path] = true } func (mc *fileGroupInfoCollector) AddDir(dirInfo *DirectoryInfo) { mc.mu.Lock() defer mc.mu.Unlock() mc.Dirs = append(mc.Dirs, dirInfo) + mc.seen[dirInfo.Path] = true +} + +func (mc *fileGroupInfoCollector) AddSymlink(symlink *SymlinkInfo) { + mc.mu.Lock() + defer mc.mu.Unlock() + mc.Symlinks = append(mc.Symlinks, symlink) + mc.seen[symlink.Path] = true +} + +func (mc *fileGroupInfoCollector) isSeen(path string) bool { + mc.mu.Lock() + defer mc.mu.Unlock() + + return mc.seen[path] } -func collectFileGroupInfo(cacheDirPath string, rootDir string, collectAttributes bool, logger log.Logger) (FileGroupInfo, error) { +func collectFileGroupInfo(cacheDirPath string, collectAttributes, followSymlinks bool, logger log.Logger) (FileGroupInfo, error) { var dd FileGroupInfo fgi := fileGroupInfoCollector{ - Files: make([]*FileInfo, 0), - Dirs: make([]*DirectoryInfo, 0), + Files: make([]*FileInfo, 0), + Dirs: make([]*DirectoryInfo, 0), + Symlinks: make([]*SymlinkInfo, 0), + seen: make(map[string]bool), } var wg sync.WaitGroup semaphore := make(chan struct{}, 10) // Limit parallelization @@ -80,7 +110,17 @@ func collectFileGroupInfo(cacheDirPath string, rootDir string, collectAttributes defer wg.Done() defer func() { <-semaphore }() // Release a slot in the semaphore - if err := collectFileMetadata(path, d, &fgi, collectAttributes, rootDir, logger); err != nil { + inf, err := d.Info() + if err != nil { + logger.Errorf("get file info: %v", err) + + return + } + + if !filepath.IsAbs(path) { + path = filepath.Join(cacheDirPath, path) + } + if err := collectFileMetadata(path, inf, inf.IsDir(), &fgi, collectAttributes, followSymlinks, logger); err != nil { logger.Errorf("Failed to collect metadata: %s", err) } }(d) @@ -96,6 +136,7 @@ func collectFileGroupInfo(cacheDirPath string, rootDir string, collectAttributes dd.Files = fgi.Files dd.Directories = fgi.Dirs + dd.Symlinks = fgi.Symlinks logger.Infof("(i) Collected %d files and %d directories ", len(dd.Files), len(dd.Directories)) logger.Debugf("(i) Largest processed file size: %s", humanize.Bytes(uint64(fgi.LargestFileSize))) @@ -103,29 +144,86 @@ func collectFileGroupInfo(cacheDirPath string, rootDir string, collectAttributes return dd, nil } -func collectFileMetadata(path string, d fs.DirEntry, fgi *fileGroupInfoCollector, collectAttributes bool, rootDir string, logger log.Logger) error { - absPath, err := filepath.Abs(path) - if err != nil { - return fmt.Errorf("get absolute path: %w", err) +// nolint:wrapcheck +func followSymlink(path string, target string, fgi *fileGroupInfoCollector, followSymlinks bool, logger log.Logger) error { + if !followSymlinks { + logger.Debugf("Skipping symbolic link: %s", path) + + return nil + } + if fgi.isSeen(target) { + logger.Debugf("Skipping symbolic link target: %s, already seen", target) + + return nil } - inf, err := d.Info() + + // Dont save symlink if target doesn't exist + stat, err := os.Stat(target) if err != nil { - return fmt.Errorf("get file info: %w", err) + return fmt.Errorf("stat target: %w", err) + } + + logger.Debugf("Resolved symlink %s to target: %s", path, target) + + fgi.AddSymlink(&SymlinkInfo{ + Path: path, + Target: target, + ModTime: stat.ModTime(), + }) + + if !stat.IsDir() { + return collectFileMetadata(target, stat, false, fgi, false, followSymlinks, logger) } - if d.IsDir() { + logger.Debugf("Symlink target is a directory, walking it: %s", target) + // Recursively walk the target directory, as it will not be included in this walk + return filepath.WalkDir(target, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return fmt.Errorf("walk target dir: %w", err) + } + + inf, err := d.Info() + if err != nil { + return fmt.Errorf("get file info: %w", err) + } + + if !filepath.IsAbs(path) { + path = filepath.Join(target, path) + } + + return collectFileMetadata(path, inf, inf.IsDir(), fgi, false, followSymlinks, logger) + }) +} + +func collectFileMetadata(path string, fileInfo fs.FileInfo, isDirectory bool, fgi *fileGroupInfoCollector, collectAttributes, followSymlinks bool, logger log.Logger) error { + if fgi.isSeen(path) { + logger.Debugf("Skipping path %s, already seen", path) + + return nil + } + + if isDirectory { fgi.AddDir(&DirectoryInfo{ - Path: absPath, - ModTime: inf.ModTime(), + Path: path, + ModTime: fileInfo.ModTime(), }) return nil } - if inf.Mode()&os.ModeSymlink != 0 { - logger.Debugf("Skipping symbolic link: %s", path) + isSymlink := fileInfo.Mode()&os.ModeSymlink != 0 - return nil + if isSymlink { + var target string + target, err := os.Readlink(path) + if err != nil { + return fmt.Errorf("read symlink: %w", err) + } + if !filepath.IsAbs(target) { + target = filepath.Join(filepath.Dir(path), target) + } + + return followSymlink(path, target, fgi, followSymlinks, logger) } file, err := os.Open(path) @@ -140,15 +238,6 @@ func collectFileMetadata(path string, d fs.DirEntry, fgi *fileGroupInfoCollector } hash := hex.EncodeToString(hasher.Sum(nil)) - savedPath := absPath - if rootDir != "" { - relPath, err := filepath.Rel(rootDir, path) - if err != nil { - return fmt.Errorf("get relative path: %w", err) - } - savedPath = relPath - } - var attrs map[string]string if collectAttributes { attrs, err = getAttributes(path) @@ -158,11 +247,11 @@ func collectFileMetadata(path string, d fs.DirEntry, fgi *fileGroupInfoCollector } fgi.AddFile(&FileInfo{ - Path: savedPath, - Size: inf.Size(), + Path: path, + Size: fileInfo.Size(), Hash: hash, - ModTime: inf.ModTime(), - Mode: inf.Mode(), + ModTime: fileInfo.ModTime(), + Mode: fileInfo.Mode(), Attributes: attrs, }) @@ -197,6 +286,38 @@ func setAttributes(path string, attributes map[string]string) error { return nil } +func restoreSymlink(symlink SymlinkInfo, logger log.Logger) bool { + fileInfo, err := os.Lstat(symlink.Path) + if err == nil && fileInfo.Mode()&os.ModeSymlink != 0 { + logger.Debugf("Symlink %s already exists, overwriting...", symlink.Path) + + err := os.Remove(symlink.Path) + if err != nil { + logger.Infof("Error removing existing symlink %s: %v", symlink.Path, err) + + return false + } + } + + err = os.Symlink(symlink.Target, symlink.Path) + if err != nil { + logger.Debugf("Error creating symlink %s -> %s: %v", symlink.Path, symlink.Target, err) + + return false + } + + // Set times + mtimeSpec := syscall.NsecToTimespec(symlink.ModTime.UnixNano()) + err = syscall.UtimesNano(symlink.Path, []syscall.Timespec{mtimeSpec, mtimeSpec}) + if err != nil { + logger.Debugf("Error setting symlink times for %s: %v", symlink.Path, err) + + return false + } + + return true +} + func restoreFileInfo(fi FileInfo, rootDir string, logger log.Logger) bool { var path string if filepath.IsAbs(fi.Path) { @@ -230,7 +351,7 @@ func restoreFileInfo(fi FileInfo, rootDir string, logger log.Logger) bool { } if err = os.Chmod(fi.Path, fi.Mode); err != nil { - logger.Debugf("Error setting file mode time for %s: %v", fi.Path, err) + logger.Debugf("Error setting file mode for %s: %v", fi.Path, err) return false } diff --git a/internal/xcode/metadata.go b/internal/xcode/metadata.go index 3cae9b85..3dfa2e35 100644 --- a/internal/xcode/metadata.go +++ b/internal/xcode/metadata.go @@ -32,6 +32,7 @@ type CreateMetadataParams struct { DerivedDataPath string XcodeCacheDirPath string CacheKey string + FollowSymlinks bool } func CreateMetadata(params CreateMetadataParams, envProvider func(string) string, logger log.Logger) (*Metadata, error) { @@ -39,14 +40,20 @@ func CreateMetadata(params CreateMetadataParams, envProvider func(string) string return nil, fmt.Errorf("missing project root directory path") } var projectFiles FileGroupInfo - projectFiles, err := collectFileGroupInfo(params.ProjectRootDirPath, params.ProjectRootDirPath, true, logger) + projectFiles, err := collectFileGroupInfo(params.ProjectRootDirPath, + true, + params.FollowSymlinks, + logger) if err != nil { return nil, fmt.Errorf("calculate project files info: %w", err) } var derivedData FileGroupInfo if params.DerivedDataPath != "" { - derivedData, err = collectFileGroupInfo(params.DerivedDataPath, "", false, logger) + derivedData, err = collectFileGroupInfo(params.DerivedDataPath, + false, + params.FollowSymlinks, + logger) if err != nil { return nil, fmt.Errorf("calculate derived data info: %w", err) } @@ -54,7 +61,10 @@ func CreateMetadata(params CreateMetadataParams, envProvider func(string) string var xcodeCacheDir FileGroupInfo if params.XcodeCacheDirPath != "" { - xcodeCacheDir, err = collectFileGroupInfo(params.XcodeCacheDirPath, "", false, logger) + xcodeCacheDir, err = collectFileGroupInfo(params.XcodeCacheDirPath, + false, + params.FollowSymlinks, + logger) if err != nil { return nil, fmt.Errorf("calculate xcode cache dir info: %w", err) } @@ -133,6 +143,22 @@ func RestoreDirectoryInfos(dirInfos []*DirectoryInfo, rootDir string, logger log return nil } +func RestoreSymlinks(symlinks []*SymlinkInfo, logger log.Logger) (int, error) { + updated := 0 + + logger.Infof("(i) %d symlinks' info loaded from cache metadata", len(symlinks)) + + for _, si := range symlinks { + if restoreSymlink(*si, logger) { + updated++ + } + } + + logger.Infof("(i) %d symlinks restored", updated) + + return updated, nil +} + func RestoreFileInfos(fileInfos []*FileInfo, rootDir string, logger log.Logger) (int, error) { updated := 0 diff --git a/internal/xcode/metadata_test.go b/internal/xcode/metadata_test.go index 48da711d..cc45ff16 100644 --- a/internal/xcode/metadata_test.go +++ b/internal/xcode/metadata_test.go @@ -66,7 +66,7 @@ func Test_CreateMetadata(t *testing.T) { require.Len(t, md.ProjectFiles.Files, 1) fi := md.ProjectFiles.Files[0] - require.True(t, strings.HasPrefix(fi.Path, "test-file.swift")) + require.True(t, strings.Contains(fi.Path, "test-file.swift")) require.NotEmpty(t, fi.Hash) require.NotEmpty(t, md.CacheKey)