From 123f135a649f27aa3a31bf1839f89b4131ce86d7 Mon Sep 17 00:00:00 2001 From: Greg Soltis Date: Fri, 7 Oct 2022 10:16:52 -0700 Subject: [PATCH] Revert "Caching, but to tar files. (#1991)" This reverts commit 4ec2013d9001c8d9c0c2de83dc17a6c9e67ee540. --- cli/go.mod | 3 +- cli/go.sum | 2 - cli/internal/cache/cache_fs.go | 85 +- cli/internal/cache/cache_fs_test.go | 106 +- cli/internal/cache/cache_http.go | 4 +- cli/internal/cacheitem/cacheitem.go | 77 - cli/internal/cacheitem/create.go | 111 -- cli/internal/cacheitem/create_test.go | 198 --- cli/internal/cacheitem/create_unix_test.go | 20 - cli/internal/cacheitem/create_windows_test.go | 14 - cli/internal/cacheitem/filepath.go | 162 -- cli/internal/cacheitem/filepath_unix.go | 14 - cli/internal/cacheitem/filepath_windows.go | 50 - cli/internal/cacheitem/restore.go | 194 --- cli/internal/cacheitem/restore_directory.go | 144 -- .../cacheitem/restore_directory_test.go | 103 -- cli/internal/cacheitem/restore_regular.go | 46 - cli/internal/cacheitem/restore_symlink.go | 180 -- cli/internal/cacheitem/restore_test.go | 1449 ----------------- cli/internal/filewatcher/cookie.go | 2 +- cli/internal/filewatcher/filewatcher_test.go | 10 +- cli/internal/globby/globby.go | 39 +- cli/internal/globby/globby_test.go | 236 +-- .../hashing/package_deps_hash_test.go | 2 +- cli/internal/runcache/runcache.go | 2 +- .../turbopath/absolute_system_path.go | 31 +- .../turbopath/absolute_system_path_darwin.go | 23 - .../absolute_system_path_notdarwin.go | 13 - .../turbopath/anchored_system_path.go | 5 - cli/package.json | 1 - cli/scripts/e2e/e2e.ts | 14 +- pnpm-lock.yaml | 47 - 32 files changed, 148 insertions(+), 3239 deletions(-) delete mode 100644 cli/internal/cacheitem/cacheitem.go delete mode 100644 cli/internal/cacheitem/create.go delete mode 100644 cli/internal/cacheitem/create_test.go delete mode 100644 cli/internal/cacheitem/create_unix_test.go delete mode 100644 cli/internal/cacheitem/create_windows_test.go delete mode 100644 cli/internal/cacheitem/filepath.go delete mode 100644 cli/internal/cacheitem/filepath_unix.go delete mode 100644 cli/internal/cacheitem/filepath_windows.go delete mode 100644 cli/internal/cacheitem/restore.go delete mode 100644 cli/internal/cacheitem/restore_directory.go delete mode 100644 cli/internal/cacheitem/restore_directory_test.go delete mode 100644 cli/internal/cacheitem/restore_regular.go delete mode 100644 cli/internal/cacheitem/restore_symlink.go delete mode 100644 cli/internal/cacheitem/restore_test.go delete mode 100644 cli/internal/turbopath/absolute_system_path_darwin.go delete mode 100644 cli/internal/turbopath/absolute_system_path_notdarwin.go diff --git a/cli/go.mod b/cli/go.mod index ce78aeac51ee1..e18d4399d09ca 100644 --- a/cli/go.mod +++ b/cli/go.mod @@ -25,7 +25,6 @@ require ( github.com/mitchellh/cli v1.1.2 github.com/mitchellh/go-homedir v1.1.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/moby/sys/sequential v0.5.0 github.com/nightlyone/lockfile v1.0.0 github.com/pkg/errors v0.9.1 github.com/pyr-sh/dag v1.0.0 @@ -37,7 +36,6 @@ require ( github.com/stretchr/testify v1.8.0 github.com/yookoala/realpath v1.0.0 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a google.golang.org/grpc v1.46.2 google.golang.org/protobuf v1.28.0 gopkg.in/yaml.v3 v3.0.1 @@ -81,6 +79,7 @@ require ( github.com/subosito/gotenv v1.3.0 // indirect golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect golang.org/x/text v0.3.7 // indirect google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect diff --git a/cli/go.sum b/cli/go.sum index b4eb7236210e4..62ca354dbcda9 100644 --- a/cli/go.sum +++ b/cli/go.sum @@ -360,8 +360,6 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE= github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc= -github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= diff --git a/cli/internal/cache/cache_fs.go b/cli/internal/cache/cache_fs.go index 09d37070dc8f8..4bff9fc9b4764 100644 --- a/cli/internal/cache/cache_fs.go +++ b/cli/internal/cache/cache_fs.go @@ -1,29 +1,30 @@ // Adapted from https://github.com/thought-machine/please // Copyright Thought Machine, Inc. or its affiliates. All Rights Reserved. // SPDX-License-Identifier: Apache-2.0 - -// Package cache implements our cache abstraction. package cache import ( "encoding/json" "fmt" + "runtime" "github.com/vercel/turborepo/cli/internal/analytics" - "github.com/vercel/turborepo/cli/internal/cacheitem" + "github.com/vercel/turborepo/cli/internal/fs" "github.com/vercel/turborepo/cli/internal/turbopath" + "golang.org/x/sync/errgroup" ) // fsCache is a local filesystem cache type fsCache struct { cacheDirectory turbopath.AbsoluteSystemPath recorder analytics.Recorder + repoRoot turbopath.AbsoluteSystemPath } // newFsCache creates a new filesystem cache func newFsCache(opts Opts, recorder analytics.Recorder, repoRoot turbopath.AbsoluteSystemPath) (*fsCache, error) { cacheDir := opts.ResolveCacheDir(repoRoot) - if err := cacheDir.MkdirAll(0775); err != nil { + if err := cacheDir.MkdirAll(); err != nil { return nil, err } return &fsCache{ @@ -34,44 +35,33 @@ func newFsCache(opts Opts, recorder analytics.Recorder, repoRoot turbopath.Absol // Fetch returns true if items are cached. It moves them into position as a side effect. func (f *fsCache) Fetch(anchor turbopath.AbsoluteSystemPath, hash string, _unusedOutputGlobs []string) (bool, []turbopath.AnchoredSystemPath, int, error) { - cachePath := f.cacheDirectory.UntypedJoin(hash + ".tar.gz") + cachedFolder := f.cacheDirectory.UntypedJoin(hash) // If it's not in the cache bail now - if !cachePath.FileExists() { + if !cachedFolder.DirExists() { f.logFetch(false, hash, 0) return false, nil, 0, nil } - cacheItem, openErr := cacheitem.Open(cachePath) - if openErr != nil { - return false, nil, 0, openErr - } - - restoredFiles, restoreErr := cacheItem.Restore(anchor) - if restoreErr != nil { - _ = cacheItem.Close() - return false, nil, 0, restoreErr + // Otherwise, copy it into position + err := fs.RecursiveCopy(cachedFolder.ToStringDuringMigration(), anchor.ToStringDuringMigration()) + if err != nil { + // TODO: what event to log here? + return false, nil, 0, fmt.Errorf("error moving artifact from cache into %v: %w", anchor, err) } meta, err := ReadCacheMetaFile(f.cacheDirectory.UntypedJoin(hash + "-meta.json")) if err != nil { - _ = cacheItem.Close() return false, nil, 0, fmt.Errorf("error reading cache metadata: %w", err) } f.logFetch(true, hash, meta.Duration) - - // Wait to see what happens with close. - closeErr := cacheItem.Close() - if closeErr != nil { - return false, restoredFiles, 0, closeErr - } - return true, restoredFiles, meta.Duration, nil + return true, nil, meta.Duration, nil } func (f *fsCache) Exists(hash string) (ItemStatus, error) { - cachePath := f.cacheDirectory.UntypedJoin(hash + "tar.gz") + cachedFolder := f.cacheDirectory.UntypedJoin(hash) - if !cachePath.FileExists() { + if !cachedFolder.DirExists() { return ItemStatus{Local: false}, nil } @@ -95,18 +85,40 @@ func (f *fsCache) logFetch(hit bool, hash string, duration int) { } func (f *fsCache) Put(anchor turbopath.AbsoluteSystemPath, hash string, duration int, files []turbopath.AnchoredSystemPath) error { - cachePath := f.cacheDirectory.UntypedJoin(hash + ".tar.gz") - cacheItem, err := cacheitem.Create(cachePath) - if err != nil { - return err + g := new(errgroup.Group) + + numDigesters := runtime.NumCPU() + fileQueue := make(chan turbopath.AnchoredSystemPath, numDigesters) + + for i := 0; i < numDigesters; i++ { + g.Go(func() error { + for file := range fileQueue { + statedFile := fs.LstatCachedFile{Path: file.RestoreAnchor(anchor)} + fromType, err := statedFile.GetType() + if err != nil { + return fmt.Errorf("error stat'ing cache source %v: %v", file, err) + } + if !fromType.IsDir() { + if err := f.cacheDirectory.UntypedJoin(hash, file.ToStringDuringMigration()).EnsureDir(); err != nil { + return fmt.Errorf("error ensuring directory file from cache: %w", err) + } + + if err := fs.CopyFile(&statedFile, f.cacheDirectory.UntypedJoin(hash, file.ToStringDuringMigration()).ToStringDuringMigration()); err != nil { + return fmt.Errorf("error copying file from cache: %w", err) + } + } + } + return nil + }) } for _, file := range files { - err := cacheItem.AddFile(anchor, file) - if err != nil { - _ = cacheItem.Close() - return err - } + fileQueue <- file + } + close(fileQueue) + + if err := g.Wait(); err != nil { + return err } writeErr := WriteCacheMetaFile(f.cacheDirectory.UntypedJoin(hash+"-meta.json"), &CacheMetadata{ @@ -115,11 +127,10 @@ func (f *fsCache) Put(anchor turbopath.AbsoluteSystemPath, hash string, duration }) if writeErr != nil { - _ = cacheItem.Close() return writeErr } - return cacheItem.Close() + return nil } func (f *fsCache) Clean(anchor turbopath.AbsoluteSystemPath) { @@ -130,7 +141,7 @@ func (f *fsCache) CleanAll() { fmt.Println("Not implemented yet") } -func (f *fsCache) Shutdown() {} +func (cache *fsCache) Shutdown() {} // CacheMetadata stores duration and hash information for a cache entry so that aggregate Time Saved calculations // can be made from artifacts from various caches diff --git a/cli/internal/cache/cache_fs_test.go b/cli/internal/cache/cache_fs_test.go index f74e0311e09bf..0039a86b9ff45 100644 --- a/cli/internal/cache/cache_fs_test.go +++ b/cli/internal/cache/cache_fs_test.go @@ -1,11 +1,12 @@ package cache import ( + "io/ioutil" + "os" "path/filepath" "testing" "github.com/vercel/turborepo/cli/internal/analytics" - "github.com/vercel/turborepo/cli/internal/cacheitem" "github.com/vercel/turborepo/cli/internal/turbopath" "gotest.tools/v3/assert" ) @@ -32,7 +33,7 @@ func TestPut(t *testing.T) { src := turbopath.AbsoluteSystemPath(t.TempDir()) childDir := src.UntypedJoin("child") - err := childDir.MkdirAll(0775) + err := childDir.MkdirAll() assert.NilError(t, err, "Mkdir") aPath := childDir.UntypedJoin("a") aFile, err := aPath.Create() @@ -58,6 +59,7 @@ func TestPut(t *testing.T) { assert.NilError(t, circlePath.Symlink(filepath.FromSlash("../child")), "Symlink") files := []turbopath.AnchoredSystemPath{ + turbopath.AnchoredUnixPath(".").ToSystemPath(), // src turbopath.AnchoredUnixPath("child/").ToSystemPath(), // childDir turbopath.AnchoredUnixPath("child/a").ToSystemPath(), // aPath, turbopath.AnchoredUnixPath("b").ToSystemPath(), // bPath, @@ -76,26 +78,17 @@ func TestPut(t *testing.T) { hash := "the-hash" duration := 0 - putErr := cache.Put(src, hash, duration, files) - assert.NilError(t, putErr, "Put") + err = cache.Put(src, hash, duration, files) + assert.NilError(t, err, "Put") // Verify that we got the files that we're expecting dstCachePath := dst.UntypedJoin(hash) - // This test checks outputs, so we go ahead and pull things back out. - // Attempting to satisfy our beliefs that the change is viable with - // as few changes to the tests as possible. - cacheItem, openErr := cacheitem.Open(dst.UntypedJoin(hash + ".tar.gz")) - assert.NilError(t, openErr, "Open") - - _, restoreErr := cacheItem.Restore(dstCachePath) - assert.NilError(t, restoreErr, "Restore") - dstAPath := dstCachePath.UntypedJoin("child", "a") - assertFileMatches(t, aPath, dstAPath) + assertFileMatches(t, aPath.ToStringDuringMigration(), dstAPath.ToStringDuringMigration()) dstBPath := dstCachePath.UntypedJoin("b") - assertFileMatches(t, bPath, dstBPath) + assertFileMatches(t, bPath.ToStringDuringMigration(), dstBPath.ToStringDuringMigration()) dstLinkPath := dstCachePath.UntypedJoin("child", "link") target, err := dstLinkPath.Readlink() @@ -118,20 +111,18 @@ func TestPut(t *testing.T) { if circleLinkDest != expectedCircleLinkDest { t.Errorf("Cache link got %v, want %v", circleLinkDest, expectedCircleLinkDest) } - - assert.NilError(t, cacheItem.Close(), "Close") } -func assertFileMatches(t *testing.T, orig turbopath.AbsoluteSystemPath, copy turbopath.AbsoluteSystemPath) { +func assertFileMatches(t *testing.T, orig string, copy string) { t.Helper() - origBytes, err := orig.ReadFile() + origBytes, err := ioutil.ReadFile(orig) assert.NilError(t, err, "ReadFile") - copyBytes, err := copy.ReadFile() + copyBytes, err := ioutil.ReadFile(copy) assert.NilError(t, err, "ReadFile") assert.DeepEqual(t, origBytes, copyBytes) - origStat, err := orig.Lstat() + origStat, err := os.Lstat(orig) assert.NilError(t, err, "Lstat") - copyStat, err := copy.Lstat() + copyStat, err := os.Lstat(copy) assert.NilError(t, err, "Lstat") assert.Equal(t, origStat.Mode(), copyStat.Mode()) } @@ -157,13 +148,12 @@ func TestFetch(t *testing.T) { // "some-package"/... cacheDir := turbopath.AbsoluteSystemPath(t.TempDir()) - hash := "the-hash" - src := cacheDir.UntypedJoin(hash, "some-package") - err := src.MkdirAll(0775) + src := cacheDir.UntypedJoin("the-hash", "some-package") + err := src.MkdirAll() assert.NilError(t, err, "mkdirAll") childDir := src.UntypedJoin("child") - err = childDir.MkdirAll(0775) + err = childDir.MkdirAll() assert.NilError(t, err, "Mkdir") aPath := childDir.UntypedJoin("a") aFile, err := aPath.Create() @@ -184,12 +174,9 @@ func TestFetch(t *testing.T) { assert.NilError(t, srcLinkPath.Symlink(linkTarget), "Symlink") srcBrokenLinkPath := childDir.UntypedJoin("broken") - srcBrokenLinkTarget := turbopath.AnchoredUnixPath("missing").ToSystemPath() - assert.NilError(t, srcBrokenLinkPath.Symlink(srcBrokenLinkTarget.ToString()), "Symlink") - + assert.NilError(t, srcBrokenLinkPath.Symlink("missing"), "Symlink") circlePath := childDir.Join("circle") - srcCircleLinkTarget := turbopath.AnchoredUnixPath("../child").ToSystemPath() - assert.NilError(t, circlePath.Symlink(srcCircleLinkTarget.ToString()), "Symlink") + assert.NilError(t, circlePath.Symlink(filepath.FromSlash("../child")), "Symlink") metadataPath := cacheDir.UntypedJoin("the-hash-meta.json") err = metadataPath.WriteFile([]byte(`{"hash":"the-hash","duration":0}`), 0777) @@ -202,18 +189,6 @@ func TestFetch(t *testing.T) { recorder: dr, } - inputFiles := []turbopath.AnchoredSystemPath{ - turbopath.AnchoredUnixPath("some-package/child/").ToSystemPath(), // childDir - turbopath.AnchoredUnixPath("some-package/child/a").ToSystemPath(), // aPath, - turbopath.AnchoredUnixPath("some-package/b").ToSystemPath(), // bPath, - turbopath.AnchoredUnixPath("some-package/child/link").ToSystemPath(), // srcLinkPath, - turbopath.AnchoredUnixPath("some-package/child/broken").ToSystemPath(), // srcBrokenLinkPath, - turbopath.AnchoredUnixPath("some-package/child/circle").ToSystemPath(), // circlePath - } - - putErr := cache.Put(cacheDir.UntypedJoin(hash), hash, 0, inputFiles) - assert.NilError(t, putErr, "Put") - outputDir := turbopath.AbsoluteSystemPath(t.TempDir()) dstOutputPath := "some-package" hit, files, _, err := cache.Fetch(outputDir, "the-hash", []string{}) @@ -221,32 +196,39 @@ func TestFetch(t *testing.T) { if !hit { t.Error("Fetch got false, want true") } - if len(files) != len(inputFiles) { - t.Errorf("len(files) got %v, want %v", len(files), len(inputFiles)) + if len(files) != 0 { + // Not for any particular reason, but currently the fs cache doesn't return the + // list of files copied + t.Errorf("len(files) got %v, want 0", len(files)) } + t.Logf("files %v", files) - dstAPath := outputDir.UntypedJoin(dstOutputPath, "child", "a") - assertFileMatches(t, aPath, dstAPath) + dstAPath := filepath.Join(outputDir.ToStringDuringMigration(), dstOutputPath, "child", "a") + assertFileMatches(t, aPath.ToStringDuringMigration(), dstAPath) - dstBPath := outputDir.UntypedJoin(dstOutputPath, "b") - assertFileMatches(t, bPath, dstBPath) + dstBPath := filepath.Join(outputDir.ToStringDuringMigration(), dstOutputPath, "b") + assertFileMatches(t, bPath.ToStringDuringMigration(), dstBPath) - dstLinkPath := outputDir.UntypedJoin(dstOutputPath, "child", "link") - target, err := dstLinkPath.Readlink() + dstLinkPath := filepath.Join(outputDir.ToStringDuringMigration(), dstOutputPath, "child", "link") + target, err := os.Readlink(dstLinkPath) assert.NilError(t, err, "Readlink") if target != linkTarget { t.Errorf("Readlink got %v, want %v", target, linkTarget) } - // Assert that we restore broken symlinks correctly - dstBrokenLinkPath := outputDir.UntypedJoin(dstOutputPath, "child", "broken") - target, readlinkErr := dstBrokenLinkPath.Readlink() - assert.NilError(t, readlinkErr, "Readlink") - assert.Equal(t, target, srcBrokenLinkTarget.ToString()) - - // Assert that we restore symlinks to directories correctly - dstCirclePath := outputDir.UntypedJoin(dstOutputPath, "child", "circle") - circleTarget, circleReadlinkErr := dstCirclePath.Readlink() - assert.NilError(t, circleReadlinkErr, "Circle Readlink") - assert.Equal(t, circleTarget, srcCircleLinkTarget.ToString()) + // We currently don't restore broken symlinks. This is probably a bug + dstBrokenLinkPath := filepath.Join(outputDir.ToStringDuringMigration(), dstOutputPath, "child", "broken") + _, err = os.Readlink(dstBrokenLinkPath) + assert.ErrorIs(t, err, os.ErrNotExist) + + // Currently, on restore, we convert symlink-to-directory to empty-directory + // This is very likely not ideal behavior, but leaving this test here to verify + // that it is what we expect at this point in time. + dstCirclePath := filepath.Join(outputDir.ToStringDuringMigration(), dstOutputPath, "child", "circle") + circleStat, err := os.Lstat(dstCirclePath) + assert.NilError(t, err, "Lstat") + assert.Equal(t, circleStat.IsDir(), true) + entries, err := os.ReadDir(dstCirclePath) + assert.NilError(t, err, "ReadDir") + assert.Equal(t, len(entries), 0) } diff --git a/cli/internal/cache/cache_http.go b/cli/internal/cache/cache_http.go index e30df25c1813c..14d9c977ceff1 100644 --- a/cli/internal/cache/cache_http.go +++ b/cli/internal/cache/cache_http.go @@ -289,12 +289,12 @@ func restoreTar(root turbopath.AbsoluteSystemPath, reader io.Reader) ([]turbopat } switch hdr.Typeflag { case tar.TypeDir: - if err := filename.MkdirAll(0775); err != nil { + if err := filename.MkdirAll(); err != nil { return nil, err } case tar.TypeReg: if dir := filename.Dir(); dir != "." { - if err := dir.MkdirAll(0775); err != nil { + if err := dir.MkdirAll(); err != nil { return nil, err } } diff --git a/cli/internal/cacheitem/cacheitem.go b/cli/internal/cacheitem/cacheitem.go deleted file mode 100644 index 8ac214956f5d2..0000000000000 --- a/cli/internal/cacheitem/cacheitem.go +++ /dev/null @@ -1,77 +0,0 @@ -// Package cacheitem is an abstraction over the creation and restoration of a cache -package cacheitem - -import ( - "archive/tar" - "compress/gzip" - "crypto/sha512" - "errors" - "hash" - "io" - "os" - - "github.com/vercel/turborepo/cli/internal/turbopath" -) - -var ( - errMissingSymlinkTarget = errors.New("symlink restoration is delayed") - errCycleDetected = errors.New("links in the cache are cyclic") - errTraversal = errors.New("tar attempts to write outside of directory") - errNameMalformed = errors.New("file name is malformed") - errNameWindowsUnsafe = errors.New("file name is not Windows-safe") - errUnsupportedFileType = errors.New("attempted to restore unsupported file type") -) - -// CacheItem is a `tar` utility with a little bit extra. -type CacheItem struct { - // Path is the location on disk for the CacheItem. - Path turbopath.AbsoluteSystemPath - // Anchor is the position on disk at which the CacheItem will be restored. - Anchor turbopath.AbsoluteSystemPath - - // For creation. - sha hash.Hash - tw *tar.Writer - gzw *gzip.Writer - handle *os.File -} - -// Close any open pipes -func (ci *CacheItem) Close() error { - if ci.tw != nil { - if err := ci.tw.Close(); err != nil { - return err - } - } - - if ci.gzw != nil { - if err := ci.gzw.Close(); err != nil { - return err - } - } - - if ci.handle != nil { - if err := ci.handle.Close(); err != nil { - return err - } - } - - return nil -} - -// GetSha returns the SHA-512 hash for the CacheItem. -func (ci *CacheItem) GetSha() ([]byte, error) { - if ci.sha != nil { - return ci.sha.Sum(nil), nil - } - - sha := sha512.New() - if _, err := io.Copy(sha, ci.handle); err != nil { - return nil, err - } - - // Don't mutate the sha until it will return the correct value. - ci.sha = sha - - return ci.sha.Sum(nil), nil -} diff --git a/cli/internal/cacheitem/create.go b/cli/internal/cacheitem/create.go deleted file mode 100644 index 57a9fef015d5c..0000000000000 --- a/cli/internal/cacheitem/create.go +++ /dev/null @@ -1,111 +0,0 @@ -package cacheitem - -import ( - "archive/tar" - "compress/gzip" - "crypto/sha512" - "io" - "os" - "time" - - "github.com/moby/sys/sequential" - "github.com/vercel/turborepo/cli/internal/tarpatch" - "github.com/vercel/turborepo/cli/internal/turbopath" -) - -// Create makes a new CacheItem at the specified path. -func Create(path turbopath.AbsoluteSystemPath) (*CacheItem, error) { - handle, err := path.Create() - if err != nil { - return nil, err - } - - cacheItem := &CacheItem{ - Path: path, - handle: handle, - } - - cacheItem.init() - return cacheItem, nil -} - -// init prepares the CacheItem for writing. -// Wires all the writers end-to-end: -// tar.Writer -> gzip.Writer -> io.MultiWriter -> (file & sha) -func (ci *CacheItem) init() { - sha := sha512.New() - mw := io.MultiWriter(sha, ci.handle) - gzw := gzip.NewWriter(mw) - tw := tar.NewWriter(gzw) - - ci.tw = tw - ci.gzw = gzw - ci.sha = sha -} - -// AddFile adds a user-cached item to the tar. -func (ci *CacheItem) AddFile(fsAnchor turbopath.AbsoluteSystemPath, filePath turbopath.AnchoredSystemPath) error { - // Calculate the fully-qualified path to the file to read it. - sourcePath := filePath.RestoreAnchor(fsAnchor) - - // We grab the FileInfo which tar.FileInfoHeader accepts. - fileInfo, lstatErr := sourcePath.Lstat() - if lstatErr != nil { - return lstatErr - } - - // Determine if we need to populate the additional link argument to tar.FileInfoHeader. - var link string - if fileInfo.Mode()&os.ModeSymlink != 0 { - linkTarget, readlinkErr := sourcePath.Readlink() - if readlinkErr != nil { - return readlinkErr - } - link = linkTarget - } - - // Normalize the path within the cache. - cacheDestinationName := filePath.ToUnixPath() - - // Generate the the header. - // We do not use header generation from stdlib because it can throw an error. - header, headerErr := tarpatch.FileInfoHeader(cacheDestinationName, fileInfo, link) - if headerErr != nil { - return headerErr - } - - // Throw an error if trying to create a cache that contains a type we don't support. - if (header.Typeflag != tar.TypeReg) && (header.Typeflag != tar.TypeDir) && (header.Typeflag != tar.TypeSymlink) { - return errUnsupportedFileType - } - - // Consistent creation. - header.Uid = 0 - header.Gid = 0 - header.AccessTime = time.Unix(0, 0) - header.ModTime = time.Unix(0, 0) - header.ChangeTime = time.Unix(0, 0) - - // Always write the header. - if err := ci.tw.WriteHeader(header); err != nil { - return err - } - - // If there is a body to be written, do so. - if header.Typeflag == tar.TypeReg && header.Size > 0 { - // Windows has a distinct "sequential read" opening mode. - // We use a library that will switch to this mode for Windows. - sourceFile, sourceErr := sequential.Open(sourcePath.ToString()) - if sourceErr != nil { - return sourceErr - } - - if _, err := io.Copy(ci.tw, sourceFile); err != nil { - return err - } - - return sourceFile.Close() - } - - return nil -} diff --git a/cli/internal/cacheitem/create_test.go b/cli/internal/cacheitem/create_test.go deleted file mode 100644 index 806f2f152bf36..0000000000000 --- a/cli/internal/cacheitem/create_test.go +++ /dev/null @@ -1,198 +0,0 @@ -package cacheitem - -import ( - "encoding/hex" - "io/fs" - "os" - "runtime" - "testing" - - "github.com/vercel/turborepo/cli/internal/turbopath" - "gotest.tools/v3/assert" -) - -type createFileDefinition struct { - Path turbopath.AnchoredSystemPath - Linkname string - fs.FileMode -} - -func createEntry(t *testing.T, anchor turbopath.AbsoluteSystemPath, fileDefinition createFileDefinition) error { - t.Helper() - if fileDefinition.FileMode.IsDir() { - return createDir(t, anchor, fileDefinition) - } else if fileDefinition.FileMode&os.ModeSymlink != 0 { - return createSymlink(t, anchor, fileDefinition) - } else if fileDefinition.FileMode&os.ModeNamedPipe != 0 { - return createFifo(t, anchor, fileDefinition) - } else { - return createFile(t, anchor, fileDefinition) - } -} - -func createDir(t *testing.T, anchor turbopath.AbsoluteSystemPath, fileDefinition createFileDefinition) error { - t.Helper() - path := fileDefinition.Path.RestoreAnchor(anchor) - mkdirAllErr := path.MkdirAllMode(fileDefinition.FileMode & 0777) - assert.NilError(t, mkdirAllErr, "MkdirAll") - return mkdirAllErr -} -func createFile(t *testing.T, anchor turbopath.AbsoluteSystemPath, fileDefinition createFileDefinition) error { - t.Helper() - path := fileDefinition.Path.RestoreAnchor(anchor) - writeErr := path.WriteFile([]byte("file contents"), fileDefinition.FileMode&0777) - assert.NilError(t, writeErr, "WriteFile") - return writeErr -} -func createSymlink(t *testing.T, anchor turbopath.AbsoluteSystemPath, fileDefinition createFileDefinition) error { - t.Helper() - path := fileDefinition.Path.RestoreAnchor(anchor) - symlinkErr := path.Symlink(fileDefinition.Linkname) - assert.NilError(t, symlinkErr, "Symlink") - lchmodErr := path.Lchmod(fileDefinition.FileMode & 0777) - assert.NilError(t, lchmodErr, "Lchmod") - return symlinkErr -} - -func TestCreate(t *testing.T) { - tests := []struct { - name string - files []createFileDefinition - wantDarwin string - wantUnix string - wantWindows string - wantErr error - }{ - { - name: "hello world", - files: []createFileDefinition{ - { - Path: turbopath.AnchoredSystemPath("hello world.txt"), - FileMode: 0 | 0644, - }, - }, - wantDarwin: "ac50a36fbd1c77ebe270bb1a383da5b1a5cf546bf9e04682ff4b2691daca5e8f16f878d6a3db179a2d1c363b4fadc98ce80645a6f820b5b399b5ac0a3c07a384", - wantUnix: "ac50a36fbd1c77ebe270bb1a383da5b1a5cf546bf9e04682ff4b2691daca5e8f16f878d6a3db179a2d1c363b4fadc98ce80645a6f820b5b399b5ac0a3c07a384", - wantWindows: "37a271d277c299cfe130ccfdb98af6e5909ade7a640a126d1495a57af1b1aed0676eedd2f0c918a9dfc04145051f52c783e7e6c0eb9aaa32af8238b47aed16bf", - }, - { - name: "links", - files: []createFileDefinition{ - { - Path: turbopath.AnchoredSystemPath("one"), - Linkname: "two", - FileMode: 0 | os.ModeSymlink | 0777, - }, - { - Path: turbopath.AnchoredSystemPath("two"), - Linkname: "three", - FileMode: 0 | os.ModeSymlink | 0777, - }, - { - Path: turbopath.AnchoredSystemPath("three"), - Linkname: "real", - FileMode: 0 | os.ModeSymlink | 0777, - }, - { - Path: turbopath.AnchoredSystemPath("real"), - FileMode: 0 | 0644, - }, - }, - wantDarwin: "3ef6504edc2865b89afe7aa07c181425c79a7f4193786792bc56a58c70cfc9cf4b8486f8af868c58894ba05ea2133893ad6a0de5d1f488cd0c5ad2ca8fc96204", - wantUnix: "3ef6504edc2865b89afe7aa07c181425c79a7f4193786792bc56a58c70cfc9cf4b8486f8af868c58894ba05ea2133893ad6a0de5d1f488cd0c5ad2ca8fc96204", - wantWindows: "46f4e6053867da99065e758c05648d2c5025830bf1e2fc9d54af1835e1d9ef3359f9a0ec942a4c8f88ebe427460cc92d2d9661956787b6045eb7ac9ecab4b5be", - }, - { - name: "subdirectory", - files: []createFileDefinition{ - { - Path: turbopath.AnchoredSystemPath("parent"), - FileMode: 0 | os.ModeDir | 0755, - }, - { - Path: turbopath.AnchoredSystemPath("parent/child"), - FileMode: 0 | 0644, - }, - }, - wantDarwin: "b8919559a95f229b9d0a460882566fee5cdd824388ecb6ef1a65938d1172ca1678ea054a0079a93ab58f041a78e3f35c911ed622a8d6c39d768299aa7f349cfa", - wantUnix: "b8919559a95f229b9d0a460882566fee5cdd824388ecb6ef1a65938d1172ca1678ea054a0079a93ab58f041a78e3f35c911ed622a8d6c39d768299aa7f349cfa", - wantWindows: "59201a55277cf9182d3513110eae0391c3881e441fcb9ec7a22d4d1e7e4c640568b29fa1ece502791ab15a1415a21e861a36c5b93c9544d675e71f0d3a613909", - }, - { - name: "symlink permissions", - files: []createFileDefinition{ - { - Path: turbopath.AnchoredSystemPath("one"), - Linkname: "two", - FileMode: 0 | os.ModeSymlink | 0644, - }, - }, - wantDarwin: "70bcf2ca3437520f4283797a93311bde0b8b8e13abd03bd7409eead9c1526a84306ca57d8a10a8027ffa64c53a134ceaddda664df87bdbeded3c05a1e98d8688", - wantUnix: "c07abb37f1bcf96e1edf5e1c45d58186475d1451eb0cc0fb906a7cef013800d5005855be1998da067c67a6f8a27c7187d7eeafd2a50ad93f8088d9f44e2202e7", - wantWindows: "8d83add4152804c50bafa2779160cbd93d4e4d29deffa48526600291ba0b973c3a56e6adcc1eaa1e26dd18c352929279341db088841eccfa27e3ab37916961da", - }, - { - name: "unsupported types error", - files: []createFileDefinition{ - { - Path: turbopath.AnchoredSystemPath("fifo"), - FileMode: 0 | os.ModeNamedPipe | 0644, - }, - }, - wantErr: errUnsupportedFileType, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - inputDir := turbopath.AbsoluteSystemPath(t.TempDir()) - archiveDir := turbopath.AbsoluteSystemPath(t.TempDir()) - archivePath := turbopath.AnchoredSystemPath("out.tar.gz").RestoreAnchor(archiveDir) - - cacheItem, cacheCreateErr := Create(archivePath) - assert.NilError(t, cacheCreateErr, "Cache Create") - - for _, file := range tt.files { - createErr := createEntry(t, inputDir, file) - if createErr != nil { - assert.ErrorIs(t, createErr, tt.wantErr) - assert.NilError(t, cacheItem.Close(), "Close") - return - } - - addFileError := cacheItem.AddFile(inputDir, file.Path) - if addFileError != nil { - assert.ErrorIs(t, addFileError, tt.wantErr) - assert.NilError(t, cacheItem.Close(), "Close") - return - } - } - - assert.NilError(t, cacheItem.Close(), "Cache Close") - - // We actually only need to compare the generated SHA. - // That ensures we got the same output. (Effectively snapshots.) - // This must be called after `Close` because both `tar` and `gzip` have footers. - shaOne, shaOneErr := cacheItem.GetSha() - assert.NilError(t, shaOneErr, "GetSha") - snapshot := hex.EncodeToString(shaOne) - - openedCacheItem, openedCacheItemErr := Open(archivePath) - assert.NilError(t, openedCacheItemErr, "Cache Open") - - shaTwo, shaTwoErr := openedCacheItem.GetSha() - snapshotTwo := hex.EncodeToString(shaTwo) - assert.NilError(t, shaTwoErr, "GetSha") - - switch runtime.GOOS { - case "darwin": - assert.Equal(t, snapshot, tt.wantDarwin, "Got expected hash.") - case "windows": - assert.Equal(t, snapshot, tt.wantWindows, "Got expected hash.") - default: - assert.Equal(t, snapshot, tt.wantUnix, "Got expected hash.") - } - assert.Equal(t, snapshot, snapshotTwo, "Reopened snapshot matches.") - assert.NilError(t, openedCacheItem.Close(), "Close") - }) - } -} diff --git a/cli/internal/cacheitem/create_unix_test.go b/cli/internal/cacheitem/create_unix_test.go deleted file mode 100644 index 7e11eb24a1a8a..0000000000000 --- a/cli/internal/cacheitem/create_unix_test.go +++ /dev/null @@ -1,20 +0,0 @@ -//go:build darwin || linux -// +build darwin linux - -package cacheitem - -import ( - "syscall" - "testing" - - "github.com/vercel/turborepo/cli/internal/turbopath" - "gotest.tools/v3/assert" -) - -func createFifo(t *testing.T, anchor turbopath.AbsoluteSystemPath, fileDefinition createFileDefinition) error { - t.Helper() - path := fileDefinition.Path.RestoreAnchor(anchor) - fifoErr := syscall.Mknod(path.ToString(), syscall.S_IFIFO|0666, 0) - assert.NilError(t, fifoErr, "FIFO") - return fifoErr -} diff --git a/cli/internal/cacheitem/create_windows_test.go b/cli/internal/cacheitem/create_windows_test.go deleted file mode 100644 index 9d75aa9406990..0000000000000 --- a/cli/internal/cacheitem/create_windows_test.go +++ /dev/null @@ -1,14 +0,0 @@ -//go:build windows -// +build windows - -package cacheitem - -import ( - "testing" - - "github.com/vercel/turborepo/cli/internal/turbopath" -) - -func createFifo(t *testing.T, anchor turbopath.AbsoluteSystemPath, fileDefinition createFileDefinition) error { - return errUnsupportedFileType -} diff --git a/cli/internal/cacheitem/filepath.go b/cli/internal/cacheitem/filepath.go deleted file mode 100644 index aa63c88b701a1..0000000000000 --- a/cli/internal/cacheitem/filepath.go +++ /dev/null @@ -1,162 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package cacheitem - -import "os" - -const _separator = os.PathSeparator - -// A lazybuf is a lazily constructed path buffer. -// It supports append, reading previously appended bytes, -// and retrieving the final string. It does not allocate a buffer -// to hold the output until that output diverges from s. -type lazybuf struct { - path string - buf []byte - w int - volAndPath string - volLen int -} - -func (b *lazybuf) index(i int) byte { - if b.buf != nil { - return b.buf[i] - } - return b.path[i] -} - -func (b *lazybuf) append(c byte) { - if b.buf == nil { - if b.w < len(b.path) && b.path[b.w] == c { - b.w++ - return - } - b.buf = make([]byte, len(b.path)) - copy(b.buf, b.path[:b.w]) - } - b.buf[b.w] = c - b.w++ -} - -func (b *lazybuf) string() string { - if b.buf == nil { - return b.volAndPath[:b.volLen+b.w] - } - return b.volAndPath[:b.volLen] + string(b.buf[:b.w]) -} - -// Clean is extracted from stdlib and removes `FromSlash` processing -// of the stdlib version. -// -// Clean returns the shortest path name equivalent to path -// by purely lexical processing. It applies the following rules -// iteratively until no further processing can be done: -// -// 1. Replace multiple Separator elements with a single one. -// 2. Eliminate each . path name element (the current directory). -// 3. Eliminate each inner .. path name element (the parent directory) -// along with the non-.. element that precedes it. -// 4. Eliminate .. elements that begin a rooted path: -// that is, replace "/.." by "/" at the beginning of a path, -// assuming Separator is '/'. -// -// The returned path ends in a slash only if it represents a root directory, -// such as "/" on Unix or `C:\` on Windows. -// -// Finally, any occurrences of slash are replaced by Separator. -// -// If the result of this process is an empty string, Clean -// returns the string ".". -// -// See also Rob Pike, ``Lexical File Names in Plan 9 or -// Getting Dot-Dot Right,'' -// https://9p.io/sys/doc/lexnames.html -func Clean(path string) string { - originalPath := path - volLen := volumeNameLen(path) - path = path[volLen:] - if path == "" { - if volLen > 1 && originalPath[1] != ':' { - // should be UNC - // ORIGINAL: return FromSlash(originalPath) - return originalPath - } - return originalPath + "." - } - rooted := os.IsPathSeparator(path[0]) - - // Invariants: - // reading from path; r is index of next byte to process. - // writing to buf; w is index of next byte to write. - // dotdot is index in buf where .. must stop, either because - // it is the leading slash or it is a leading ../../.. prefix. - n := len(path) - out := lazybuf{path: path, volAndPath: originalPath, volLen: volLen} - r, dotdot := 0, 0 - if rooted { - out.append(_separator) - r, dotdot = 1, 1 - } - - for r < n { - switch { - case os.IsPathSeparator(path[r]): - // empty path element - r++ - case path[r] == '.' && r+1 == n: - // . element - r++ - case path[r] == '.' && os.IsPathSeparator(path[r+1]): - // ./ element - r++ - - for r < len(path) && os.IsPathSeparator(path[r]) { - r++ - } - if out.w == 0 && volumeNameLen(path[r:]) > 0 { - // When joining prefix "." and an absolute path on Windows, - // the prefix should not be removed. - out.append('.') - } - case path[r] == '.' && path[r+1] == '.' && (r+2 == n || os.IsPathSeparator(path[r+2])): - // .. element: remove to last separator - r += 2 - switch { - case out.w > dotdot: - // can backtrack - out.w-- - for out.w > dotdot && !os.IsPathSeparator(out.index(out.w)) { - out.w-- - } - case !rooted: - // cannot backtrack, but not rooted, so append .. element. - if out.w > 0 { - out.append(_separator) - } - out.append('.') - out.append('.') - dotdot = out.w - } - default: - // real path element. - // add slash if needed - if rooted && out.w != 1 || !rooted && out.w != 0 { - out.append(_separator) - } - // copy element - for ; r < n && !os.IsPathSeparator(path[r]); r++ { - out.append(path[r]) - } - } - } - - // Turn empty string into "." - if out.w == 0 { - out.append('.') - } - - // ORIGINAL: return FromSlash(out.string()) - return out.string() -} diff --git a/cli/internal/cacheitem/filepath_unix.go b/cli/internal/cacheitem/filepath_unix.go deleted file mode 100644 index 5190adb040f92..0000000000000 --- a/cli/internal/cacheitem/filepath_unix.go +++ /dev/null @@ -1,14 +0,0 @@ -//go:build darwin || linux -// +build darwin linux - -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package cacheitem - -// volumeNameLen returns length of the leading volume name on Windows. -// It returns 0 elsewhere. -func volumeNameLen(path string) int { - return 0 -} diff --git a/cli/internal/cacheitem/filepath_windows.go b/cli/internal/cacheitem/filepath_windows.go deleted file mode 100644 index 2c3b852677a2b..0000000000000 --- a/cli/internal/cacheitem/filepath_windows.go +++ /dev/null @@ -1,50 +0,0 @@ -//go:build windows -// +build windows - -// Copyright 2010 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the LICENSE file. - -package cacheitem - -func isSlash(c uint8) bool { - return c == '\\' || c == '/' -} - -// volumeNameLen returns length of the leading volume name on Windows. -// It returns 0 elsewhere. -func volumeNameLen(path string) int { - if len(path) < 2 { - return 0 - } - // with drive letter - c := path[0] - if path[1] == ':' && ('a' <= c && c <= 'z' || 'A' <= c && c <= 'Z') { - return 2 - } - // is it UNC? https://msdn.microsoft.com/en-us/library/windows/desktop/aa365247(v=vs.85).aspx - if l := len(path); l >= 5 && isSlash(path[0]) && isSlash(path[1]) && - !isSlash(path[2]) && path[2] != '.' { - // first, leading `\\` and next shouldn't be `\`. its server name. - for n := 3; n < l-1; n++ { - // second, next '\' shouldn't be repeated. - if isSlash(path[n]) { - n++ - // third, following something characters. its share name. - if !isSlash(path[n]) { - if path[n] == '.' { - break - } - for ; n < l; n++ { - if isSlash(path[n]) { - break - } - } - return n - } - break - } - } - } - return 0 -} diff --git a/cli/internal/cacheitem/restore.go b/cli/internal/cacheitem/restore.go deleted file mode 100644 index 9ff2f7b0849d7..0000000000000 --- a/cli/internal/cacheitem/restore.go +++ /dev/null @@ -1,194 +0,0 @@ -package cacheitem - -import ( - "archive/tar" - "compress/gzip" - "errors" - "io" - "runtime" - "strings" - - "github.com/moby/sys/sequential" - "github.com/vercel/turborepo/cli/internal/turbopath" -) - -// Open returns an existing CacheItem at the specified path. -func Open(path turbopath.AbsoluteSystemPath) (*CacheItem, error) { - handle, err := sequential.Open(path.ToString()) - if err != nil { - return nil, err - } - - return &CacheItem{ - Path: path, - handle: handle, - }, nil -} - -// Restore extracts a cache to a specified disk location. -func (ci *CacheItem) Restore(anchor turbopath.AbsoluteSystemPath) ([]turbopath.AnchoredSystemPath, error) { - // tar wrapped in gzip, we need to stream out of gzip first. - gzr, err := gzip.NewReader(ci.handle) - if err != nil { - return nil, err - } - - // The `Close` function for compression effectively just returns the singular - // error field on the decompressor instance. This is extremely unlikely to be - // set without triggering one of the numerous other errors, but we should still - // handle that possible edge case. - var closeError error - defer func() { closeError = gzr.Close() }() - tr := tar.NewReader(gzr) - - // On first attempt to restore it's possible that a link target doesn't exist. - // Save them and topsort them. - var symlinks []*tar.Header - - restored := make([]turbopath.AnchoredSystemPath, 0) - - restorePointErr := anchor.MkdirAll(0755) - if restorePointErr != nil { - return nil, restorePointErr - } - - // We're going to make the following two assumptions here for "fast" path restoration: - // - All directories are enumerated in the `tar`. - // - The contents of the tar are enumerated depth-first. - // - // This allows us to avoid: - // - Attempts at recursive creation of directories. - // - Repetitive `lstat` on restore of a file. - // - // Violating these assumptions won't cause things to break but we're only going to maintain - // an `lstat` cache for the current tree. If you violate these assumptions and the current - // cache does not apply for your path, it will clobber and re-start from the common - // shared prefix. - dirCache := &cachedDirTree{ - anchorAtDepth: []turbopath.AbsoluteSystemPath{anchor}, - } - - for { - header, trErr := tr.Next() - if trErr == io.EOF { - // The end, time to restore any missing links. - symlinksRestored, symlinksErr := topologicallyRestoreSymlinks(dirCache, anchor, symlinks, tr) - restored = append(restored, symlinksRestored...) - if symlinksErr != nil { - return restored, symlinksErr - } - - break - } - if trErr != nil { - return restored, trErr - } - - // The reader will not advance until tr.Next is called. - // We can treat this as file metadata + body reader. - - // Attempt to place the file on disk. - file, restoreErr := restoreEntry(dirCache, anchor, header, tr) - if restoreErr != nil { - if errors.Is(restoreErr, errMissingSymlinkTarget) { - // Links get one shot to be valid, then they're accumulated, DAG'd, and restored on delay. - symlinks = append(symlinks, header) - continue - } - return restored, restoreErr - } - restored = append(restored, file) - } - - return restored, closeError -} - -// restoreRegular is the entry point for all things read from the tar. -func restoreEntry(dirCache *cachedDirTree, anchor turbopath.AbsoluteSystemPath, header *tar.Header, reader *tar.Reader) (turbopath.AnchoredSystemPath, error) { - // We're permissive on creation, but restrictive on restoration. - // There is no need to prevent the cache creation in any case. - // And on restoration, if we fail, we simply run the task. - switch header.Typeflag { - case tar.TypeDir: - return restoreDirectory(dirCache, anchor, header) - case tar.TypeReg: - return restoreRegular(dirCache, anchor, header, reader) - case tar.TypeSymlink: - return restoreSymlink(dirCache, anchor, header) - default: - return "", errUnsupportedFileType - } -} - -// canonicalizeName returns either an AnchoredSystemPath or an error. -func canonicalizeName(name string) (turbopath.AnchoredSystemPath, error) { - // Assuming this was a `turbo`-created input, we currently have an AnchoredUnixPath. - // Assuming this is malicious input we don't really care if we do the wrong thing. - wellFormed, windowsSafe := checkName(name) - - // Determine if the future filename is a well-formed AnchoredUnixPath - if !wellFormed { - return "", errNameMalformed - } - - // Determine if the AnchoredUnixPath is safe to be used on Windows - if runtime.GOOS == "windows" && !windowsSafe { - return "", errNameWindowsUnsafe - } - - // Directories will have a trailing slash. Remove it. - noTrailingSlash := strings.TrimSuffix(name, "/") - - // Okay, we're all set here. - return turbopath.AnchoredUnixPathFromUpstream(noTrailingSlash).ToSystemPath(), nil -} - -// checkName returns `wellFormed, windowsSafe` via inspection of separators and traversal -func checkName(name string) (bool, bool) { - length := len(name) - - // Name is of length 0. - if length == 0 { - return false, false - } - - wellFormed := true - windowsSafe := true - - // Name is: - // - "." - // - ".." - if wellFormed && (name == "." || name == "..") { - wellFormed = false - } - - // Name starts with: - // - `/` - // - `./` - // - `../` - if wellFormed && (strings.HasPrefix(name, "/") || strings.HasPrefix(name, "./") || strings.HasPrefix(name, "../")) { - wellFormed = false - } - - // Name ends in: - // - `/.` - // - `/..` - if wellFormed && (strings.HasSuffix(name, "/.") || strings.HasSuffix(name, "/..")) { - wellFormed = false - } - - // Name contains: - // - `//` - // - `/./` - // - `/../` - if wellFormed && (strings.Contains(name, "//") || strings.Contains(name, "/./") || strings.Contains(name, "/../")) { - wellFormed = false - } - - // Name contains: `\` - if strings.ContainsRune(name, '\\') { - windowsSafe = false - } - - return wellFormed, windowsSafe -} diff --git a/cli/internal/cacheitem/restore_directory.go b/cli/internal/cacheitem/restore_directory.go deleted file mode 100644 index 2c1365241b08b..0000000000000 --- a/cli/internal/cacheitem/restore_directory.go +++ /dev/null @@ -1,144 +0,0 @@ -package cacheitem - -import ( - "archive/tar" - "os" - "path/filepath" - "strings" - - "github.com/vercel/turborepo/cli/internal/turbopath" -) - -// restoreDirectory restores a directory. -func restoreDirectory(dirCache *cachedDirTree, anchor turbopath.AbsoluteSystemPath, header *tar.Header) (turbopath.AnchoredSystemPath, error) { - processedName, err := canonicalizeName(header.Name) - if err != nil { - return "", err - } - - // We need to traverse `processedName` from base to root split at - // `os.Separator` to make sure we don't end up following a symlink - // outside of the restore path. - - // Create the directory. - if err := safeMkdirAll(dirCache, anchor, processedName, header.Mode); err != nil { - return "", err - } - - return processedName, nil -} - -type cachedDirTree struct { - anchorAtDepth []turbopath.AbsoluteSystemPath - prefix []turbopath.RelativeSystemPath -} - -func (cr *cachedDirTree) getStartingPoint(path turbopath.AnchoredSystemPath) (turbopath.AbsoluteSystemPath, []turbopath.RelativeSystemPath) { - pathSegmentStrings := strings.Split(path.ToString(), string(os.PathSeparator)) - pathSegments := make([]turbopath.RelativeSystemPath, len(pathSegmentStrings)) - for index, pathSegmentString := range pathSegmentStrings { - pathSegments[index] = turbopath.RelativeSystemPathFromUpstream(pathSegmentString) - } - - i := 0 - for i = 0; i < len(cr.prefix) && i < len(pathSegments); i++ { - if pathSegments[i] != cr.prefix[i] { - break - } - } - - // 0: root anchor, can't remove it. - cr.anchorAtDepth = cr.anchorAtDepth[:i+1] - - // 0: first prefix. - cr.prefix = cr.prefix[:i] - - return cr.anchorAtDepth[i], pathSegments[i:] -} - -func (cr *cachedDirTree) Update(anchor turbopath.AbsoluteSystemPath, newSegment turbopath.RelativeSystemPath) { - cr.anchorAtDepth = append(cr.anchorAtDepth, anchor) - cr.prefix = append(cr.prefix, newSegment) -} - -// safeMkdirAll creates all directories, assuming that the leaf node is a directory. -// FIXME: Recheck the symlink cache before creating a directory. -func safeMkdirAll(dirCache *cachedDirTree, anchor turbopath.AbsoluteSystemPath, processedName turbopath.AnchoredSystemPath, mode int64) error { - // Iterate through path segments by os.Separator, appending them onto the anchor. - // Check to see if that path segment is a symlink with a target outside of anchor. - - // Pull the iteration starting point from thie directory cache. - calculatedAnchor, pathSegments := dirCache.getStartingPoint(processedName) - for _, segment := range pathSegments { - calculatedAnchor, checkPathErr := checkPath(anchor, calculatedAnchor, segment) - // We hit an existing directory or absolute path that was invalid. - if checkPathErr != nil { - return checkPathErr - } - - // Otherwise we continue and check the next segment. - dirCache.Update(calculatedAnchor, segment) - } - - // If we have made it here we know that it is safe to call os.MkdirAll - // on the Join of anchor and processedName. - // - // This could _still_ error, but we don't care. - return processedName.RestoreAnchor(anchor).MkdirAll(os.FileMode(mode)) -} - -// checkPath ensures that the resolved path (if restoring symlinks). -// It makes sure to never traverse outside of the anchor. -func checkPath(originalAnchor turbopath.AbsoluteSystemPath, accumulatedAnchor turbopath.AbsoluteSystemPath, segment turbopath.RelativeSystemPath) (turbopath.AbsoluteSystemPath, error) { - // Check if the segment itself is sneakily an absolute path... - // (looking at you, Windows. CON, AUX...) - if filepath.IsAbs(segment.ToString()) { - return "", errTraversal - } - - // Find out if this portion of the path is a symlink. - combinedPath := accumulatedAnchor.Join(segment) - fileInfo, err := combinedPath.Lstat() - - // Getting an error here means we failed to stat the path. - // Assume that means we're safe and continue. - if err != nil { - return combinedPath, nil - } - - // Find out if we have a symlink. - isSymlink := fileInfo.Mode()&os.ModeSymlink != 0 - - // If we don't have a symlink it's safe. - if !isSymlink { - return combinedPath, nil - } - - // Check to see if the symlink targets outside of the originalAnchor. - // We don't do eval symlinks because we could find ourself in a totally - // different place. - - // 1. Get the target. - linkTarget, readLinkErr := combinedPath.Readlink() - if readLinkErr != nil { - return "", readLinkErr - } - - // 2. See if the target is absolute. - if filepath.IsAbs(linkTarget) { - absoluteLinkTarget := turbopath.AbsoluteSystemPathFromUpstream(linkTarget) - if originalAnchor.HasPrefix(absoluteLinkTarget) { - return absoluteLinkTarget, nil - } - return "", errTraversal - } - - // 3. Target is relative (or absolute Windows on a Unix device) - relativeLinkTarget := turbopath.RelativeSystemPathFromUpstream(linkTarget) - computedTarget := accumulatedAnchor.UntypedJoin(linkTarget) - if computedTarget.HasPrefix(originalAnchor) { - // Need to recurse and make sure the target doesn't link out. - return checkPath(originalAnchor, accumulatedAnchor, relativeLinkTarget) - } - return "", errTraversal -} diff --git a/cli/internal/cacheitem/restore_directory_test.go b/cli/internal/cacheitem/restore_directory_test.go deleted file mode 100644 index a53c95c4d0721..0000000000000 --- a/cli/internal/cacheitem/restore_directory_test.go +++ /dev/null @@ -1,103 +0,0 @@ -package cacheitem - -import ( - "reflect" - "testing" - - "github.com/vercel/turborepo/cli/internal/turbopath" -) - -func Test_cachedDirTree_getStartingPoint(t *testing.T) { - testDir := turbopath.AbsoluteSystemPath("") - tests := []struct { - name string - - // STATE - cachedDirTree cachedDirTree - - // INPUT - path turbopath.AnchoredSystemPath - - // OUTPUT - calculatedAnchor turbopath.AbsoluteSystemPath - pathSegments []turbopath.RelativeSystemPath - }{ - { - name: "hello world", - cachedDirTree: cachedDirTree{ - anchorAtDepth: []turbopath.AbsoluteSystemPath{testDir}, - prefix: []turbopath.RelativeSystemPath{}, - }, - path: turbopath.AnchoredUnixPath("hello/world").ToSystemPath(), - calculatedAnchor: testDir, - pathSegments: []turbopath.RelativeSystemPath{"hello", "world"}, - }, - { - name: "has a cache", - cachedDirTree: cachedDirTree{ - anchorAtDepth: []turbopath.AbsoluteSystemPath{ - testDir, - testDir.UntypedJoin("hello"), - }, - prefix: []turbopath.RelativeSystemPath{"hello"}, - }, - path: turbopath.AnchoredUnixPath("hello/world").ToSystemPath(), - calculatedAnchor: testDir.UntypedJoin("hello"), - pathSegments: []turbopath.RelativeSystemPath{"world"}, - }, - { - name: "ask for yourself", - cachedDirTree: cachedDirTree{ - anchorAtDepth: []turbopath.AbsoluteSystemPath{ - testDir, - testDir.UntypedJoin("hello"), - testDir.UntypedJoin("hello", "world"), - }, - prefix: []turbopath.RelativeSystemPath{"hello", "world"}, - }, - path: turbopath.AnchoredUnixPath("hello/world").ToSystemPath(), - calculatedAnchor: testDir.UntypedJoin("hello", "world"), - pathSegments: []turbopath.RelativeSystemPath{}, - }, - { - name: "three layer cake", - cachedDirTree: cachedDirTree{ - anchorAtDepth: []turbopath.AbsoluteSystemPath{ - testDir, - testDir.UntypedJoin("hello"), - testDir.UntypedJoin("hello", "world"), - }, - prefix: []turbopath.RelativeSystemPath{"hello", "world"}, - }, - path: turbopath.AnchoredUnixPath("hello/world/again").ToSystemPath(), - calculatedAnchor: testDir.UntypedJoin("hello", "world"), - pathSegments: []turbopath.RelativeSystemPath{"again"}, - }, - { - name: "outside of cache hierarchy", - cachedDirTree: cachedDirTree{ - anchorAtDepth: []turbopath.AbsoluteSystemPath{ - testDir, - testDir.UntypedJoin("hello"), - testDir.UntypedJoin("hello", "world"), - }, - prefix: []turbopath.RelativeSystemPath{"hello", "world"}, - }, - path: turbopath.AnchoredUnixPath("somewhere/else").ToSystemPath(), - calculatedAnchor: testDir, - pathSegments: []turbopath.RelativeSystemPath{"somewhere", "else"}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - cr := tt.cachedDirTree - calculatedAnchor, pathSegments := cr.getStartingPoint(tt.path) - if !reflect.DeepEqual(calculatedAnchor, tt.calculatedAnchor) { - t.Errorf("cachedDirTree.getStartingPoint() calculatedAnchor = %v, want %v", calculatedAnchor, tt.calculatedAnchor) - } - if !reflect.DeepEqual(pathSegments, tt.pathSegments) { - t.Errorf("cachedDirTree.getStartingPoint() pathSegments = %v, want %v", pathSegments, tt.pathSegments) - } - }) - } -} diff --git a/cli/internal/cacheitem/restore_regular.go b/cli/internal/cacheitem/restore_regular.go deleted file mode 100644 index 3479f1bec1db4..0000000000000 --- a/cli/internal/cacheitem/restore_regular.go +++ /dev/null @@ -1,46 +0,0 @@ -package cacheitem - -import ( - "archive/tar" - "io" - "os" - - "github.com/vercel/turborepo/cli/internal/turbopath" -) - -// restoreRegular restores a file. -func restoreRegular(dirCache *cachedDirTree, anchor turbopath.AbsoluteSystemPath, header *tar.Header, reader *tar.Reader) (turbopath.AnchoredSystemPath, error) { - // Assuming this was a `turbo`-created input, we currently have an AnchoredUnixPath. - // Assuming this is malicious input we don't really care if we do the wrong thing. - processedName, err := canonicalizeName(header.Name) - if err != nil { - return "", err - } - - // We need to traverse `processedName` from base to root split at - // `os.Separator` to make sure we don't end up following a symlink - // outside of the restore path. - if err := safeMkdirFile(dirCache, anchor, processedName, header.Mode); err != nil { - return "", err - } - - // Create the file. - if f, err := processedName.RestoreAnchor(anchor).OpenFile(os.O_WRONLY|os.O_TRUNC|os.O_CREATE, os.FileMode(header.Mode)); err != nil { - return "", err - } else if _, err := io.Copy(f, reader); err != nil { - return "", err - } else if err := f.Close(); err != nil { - return "", err - } - return processedName, nil -} - -// safeMkdirAll creates all directories, assuming that the leaf node is a file. -func safeMkdirFile(dirCache *cachedDirTree, anchor turbopath.AbsoluteSystemPath, processedName turbopath.AnchoredSystemPath, mode int64) error { - isRootFile := processedName.Dir() == "." - if !isRootFile { - return safeMkdirAll(dirCache, anchor, processedName.Dir(), 0755) - } - - return nil -} diff --git a/cli/internal/cacheitem/restore_symlink.go b/cli/internal/cacheitem/restore_symlink.go deleted file mode 100644 index 7af38f13637ef..0000000000000 --- a/cli/internal/cacheitem/restore_symlink.go +++ /dev/null @@ -1,180 +0,0 @@ -package cacheitem - -import ( - "archive/tar" - "io/fs" - "os" - "path/filepath" - - "github.com/pyr-sh/dag" - "github.com/vercel/turborepo/cli/internal/turbopath" -) - -// restoreSymlink restores a symlink and errors if the target is missing. -func restoreSymlink(dirCache *cachedDirTree, anchor turbopath.AbsoluteSystemPath, header *tar.Header) (turbopath.AnchoredSystemPath, error) { - processedName, canonicalizeNameErr := canonicalizeName(header.Name) - if canonicalizeNameErr != nil { - return "", canonicalizeNameErr - } - - // Check to see if the target exists. - processedLinkname := canonicalizeLinkname(anchor, processedName, header.Linkname) - if _, err := os.Lstat(processedLinkname); err != nil { - return "", errMissingSymlinkTarget - } - - return actuallyRestoreSymlink(dirCache, anchor, processedName, header) -} - -// restoreSymlinkMissingTarget restores a symlink and does not error if the target is missing. -func restoreSymlinkMissingTarget(dirCache *cachedDirTree, anchor turbopath.AbsoluteSystemPath, header *tar.Header) (turbopath.AnchoredSystemPath, error) { - processedName, canonicalizeNameErr := canonicalizeName(header.Name) - if canonicalizeNameErr != nil { - return "", canonicalizeNameErr - } - - return actuallyRestoreSymlink(dirCache, anchor, processedName, header) -} - -func actuallyRestoreSymlink(dirCache *cachedDirTree, anchor turbopath.AbsoluteSystemPath, processedName turbopath.AnchoredSystemPath, header *tar.Header) (turbopath.AnchoredSystemPath, error) { - // We need to traverse `processedName` from base to root split at - // `os.Separator` to make sure we don't end up following a symlink - // outside of the restore path. - if err := safeMkdirFile(dirCache, anchor, processedName, header.Mode); err != nil { - return "", err - } - - // Specify where we restoring this symlink. - symlinkFrom := processedName.RestoreAnchor(anchor) - - // Remove any existing object at that location. - // If it errors we'll catch it on creation. - _ = symlinkFrom.Remove() - - // Create the symlink. - // Explicitly uses the _original_ header.Linkname as the target. - // This does not support file names with `\` in them in a cross-platform manner. - symlinkErr := symlinkFrom.Symlink(header.Linkname) - if symlinkErr != nil { - return "", symlinkErr - } - - // Darwin allows you to change the permissions of a symlink. - lchmodErr := symlinkFrom.Lchmod(fs.FileMode(header.Mode)) - if lchmodErr != nil { - return "", lchmodErr - } - - return processedName, nil -} - -// topologicallyRestoreSymlinks ensures that targets of symlinks are created in advance -// of the things that link to them. It does this by topologically sorting all -// of the symlinks. This also enables us to ensure we do not create cycles. -func topologicallyRestoreSymlinks(dirCache *cachedDirTree, anchor turbopath.AbsoluteSystemPath, symlinks []*tar.Header, tr *tar.Reader) ([]turbopath.AnchoredSystemPath, error) { - restored := make([]turbopath.AnchoredSystemPath, 0) - lookup := make(map[string]*tar.Header) - - var g dag.AcyclicGraph - for _, header := range symlinks { - processedName, err := canonicalizeName(header.Name) - processedSourcename := canonicalizeLinkname(anchor, processedName, processedName.ToString()) - processedLinkname := canonicalizeLinkname(anchor, processedName, header.Linkname) - if err != nil { - return nil, err - } - g.Add(processedSourcename) - g.Add(processedLinkname) - g.Connect(dag.BasicEdge(processedLinkname, processedSourcename)) - lookup[processedSourcename] = header - } - - cycles := g.Cycles() - if cycles != nil { - return restored, errCycleDetected - } - - roots := make(dag.Set) - for _, v := range g.Vertices() { - if g.UpEdges(v).Len() == 0 { - roots.Add(v) - } - } - - walkFunc := func(vertex dag.Vertex, depth int) error { - key, ok := vertex.(string) - if !ok { - return nil - } - header, exists := lookup[key] - if !exists { - return nil - } - - file, restoreErr := restoreSymlinkMissingTarget(dirCache, anchor, header) - if restoreErr != nil { - return restoreErr - } - - restored = append(restored, file) - return nil - } - - walkError := g.DepthFirstWalk(roots, walkFunc) - if walkError != nil { - return restored, walkError - } - - return restored, nil -} - -// canonicalizeLinkname determines (lexically) what the resolved path on the -// system will be when linkname is restored verbatim. -func canonicalizeLinkname(anchor turbopath.AbsoluteSystemPath, processedName turbopath.AnchoredSystemPath, linkname string) string { - // We don't know _anything_ about linkname. It could be any of: - // - // - Absolute Unix Path - // - Absolute Windows Path - // - Relative Unix Path - // - Relative Windows Path - // - // We also can't _truly_ distinguish if the path is Unix or Windows. - // Take for example: `/Users/turbobot/weird-filenames/\foo\/lol` - // It is a valid file on Unix, but if we do slash conversion it breaks. - // Or `i\am\a\normal\unix\file\but\super\nested\on\windows`. - // - // We also can't safely assume that paths in link targets on one platform - // should be treated as targets for that platform. The author may be - // generating an artifact that should work on Windows on a Unix device. - // - // Given all of that, our best option is to restore link targets _verbatim_. - // No modification, no slash conversion. - // - // In order to DAG sort them, however, we do need to canonicalize them. - // We canonicalize them as if we're restoring them verbatim. - // - // 0. We've extracted a version of `Clean` from stdlib which does nothing but - // separator and traversal collapsing. - cleanedLinkname := Clean(linkname) - - // 1. Check to see if the link target is absolute _on the current platform_. - // If it is an absolute path it's canonical by rule. - if filepath.IsAbs(cleanedLinkname) { - return cleanedLinkname - } - - // Remaining options: - // - Absolute (other platform) Path - // - Relative Unix Path - // - Relative Windows Path - // - // At this point we simply assume that it's a relative path—no matter - // which separators appear in it and where they appear, We can't do - // anything else because the OS will also treat it like that when it is - // a link target. - // - // We manually join these to avoid calls to stdlib's `Clean`. - source := processedName.RestoreAnchor(anchor) - canonicalized := source.Dir().ToString() + string(os.PathSeparator) + cleanedLinkname - return Clean(canonicalized) -} diff --git a/cli/internal/cacheitem/restore_test.go b/cli/internal/cacheitem/restore_test.go deleted file mode 100644 index 0feff7f22a0e8..0000000000000 --- a/cli/internal/cacheitem/restore_test.go +++ /dev/null @@ -1,1449 +0,0 @@ -package cacheitem - -import ( - "archive/tar" - "compress/gzip" - "errors" - "fmt" - "io/fs" - "os" - "path/filepath" - "reflect" - "runtime" - "syscall" - "testing" - - "github.com/vercel/turborepo/cli/internal/turbopath" - "gotest.tools/v3/assert" -) - -type tarFile struct { - Body string - *tar.Header -} - -type restoreFile struct { - Name turbopath.AnchoredUnixPath - Linkname string - fs.FileMode -} - -// This function is used specifically to generate tar files that Turborepo would -// rarely or never encounter without malicious or pathological inputs. We use it -// to make sure that we respond well in these scenarios during restore attempts. -func generateTar(t *testing.T, files []tarFile) turbopath.AbsoluteSystemPath { - t.Helper() - testDir := turbopath.AbsoluteSystemPath(t.TempDir()) - testArchivePath := testDir.UntypedJoin("out.tar.gz") - - handle, handleCreateErr := testArchivePath.Create() - assert.NilError(t, handleCreateErr, "os.Create") - - gzw := gzip.NewWriter(handle) - tw := tar.NewWriter(gzw) - - for _, file := range files { - if file.Header.Typeflag == tar.TypeReg { - file.Header.Size = int64(len(file.Body)) - } - - writeHeaderErr := tw.WriteHeader(file.Header) - assert.NilError(t, writeHeaderErr, "tw.WriteHeader") - - _, writeErr := tw.Write([]byte(file.Body)) - assert.NilError(t, writeErr, "tw.Write") - } - - twCloseErr := tw.Close() - assert.NilError(t, twCloseErr, "tw.Close") - - gzwCloseErr := gzw.Close() - assert.NilError(t, gzwCloseErr, "gzw.Close") - - handleCloseErr := handle.Close() - assert.NilError(t, handleCloseErr, "handle.Close") - - return testArchivePath -} - -func generateAnchor(t *testing.T) turbopath.AbsoluteSystemPath { - t.Helper() - testDir := turbopath.AbsoluteSystemPath(t.TempDir()) - anchorPoint := testDir.UntypedJoin("anchor") - - mkdirErr := anchorPoint.Mkdir(0777) - assert.NilError(t, mkdirErr, "Mkdir") - - return anchorPoint -} - -func assertFileExists(t *testing.T, anchor turbopath.AbsoluteSystemPath, diskFile restoreFile) { - t.Helper() - // If we have gotten here we can assume this to be true. - processedName := diskFile.Name.ToSystemPath() - fullName := processedName.RestoreAnchor(anchor) - fileInfo, err := fullName.Lstat() - assert.NilError(t, err, "Lstat") - - assert.Equal(t, fileInfo.Mode()&fs.ModePerm, diskFile.FileMode&fs.ModePerm, "File has the expected permissions: "+processedName) - assert.Equal(t, fileInfo.Mode()|fs.ModePerm, diskFile.FileMode|fs.ModePerm, "File has the expected mode.") - - if diskFile.FileMode&os.ModeSymlink != 0 { - linkname, err := fullName.Readlink() - assert.NilError(t, err, "Readlink") - - // We restore Linkname verbatim. - assert.Equal(t, linkname, diskFile.Linkname, "Link target matches.") - } -} - -func TestOpen(t *testing.T) { - type wantErr struct { - unix error - windows error - } - type wantOutput struct { - unix []turbopath.AnchoredSystemPath - windows []turbopath.AnchoredSystemPath - } - type wantFiles struct { - unix []restoreFile - windows []restoreFile - } - tests := []struct { - name string - tarFiles []tarFile - wantOutput wantOutput - wantFiles wantFiles - wantErr wantErr - }{ - { - name: "cache optimized", - tarFiles: []tarFile{ - { - Header: &tar.Header{ - Name: "one/", - Typeflag: tar.TypeDir, - Mode: 0755, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/", - Typeflag: tar.TypeDir, - Mode: 0755, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/three/", - Typeflag: tar.TypeDir, - Mode: 0755, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/three/file-one", - Typeflag: tar.TypeReg, - Mode: 0644, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/three/file-two", - Typeflag: tar.TypeReg, - Mode: 0644, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/a/", - Typeflag: tar.TypeDir, - Mode: 0755, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/a/file", - Typeflag: tar.TypeReg, - Mode: 0644, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/b/", - Typeflag: tar.TypeDir, - Mode: 0755, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/b/file", - Typeflag: tar.TypeReg, - Mode: 0644, - }, - }, - }, - wantFiles: wantFiles{ - unix: []restoreFile{ - { - Name: "one", - FileMode: 0 | os.ModeDir | 0755, - }, - { - Name: "one/two", - FileMode: 0 | os.ModeDir | 0755, - }, - { - Name: "one/two/three", - FileMode: 0 | os.ModeDir | 0755, - }, - { - Name: "one/two/three/file-one", - FileMode: 0644, - }, - { - Name: "one/two/three/file-two", - FileMode: 0644, - }, - { - Name: "one/two/a", - FileMode: 0 | os.ModeDir | 0755, - }, - { - Name: "one/two/a/file", - FileMode: 0644, - }, - { - Name: "one/two/b", - FileMode: 0 | os.ModeDir | 0755, - }, - { - Name: "one/two/b/file", - FileMode: 0644, - }, - }, - windows: []restoreFile{ - { - Name: "one", - FileMode: 0 | os.ModeDir | 0777, - }, - { - Name: "one/two", - FileMode: 0 | os.ModeDir | 0777, - }, - { - Name: "one/two/three", - FileMode: 0 | os.ModeDir | 0777, - }, - { - Name: "one/two/three/file-one", - FileMode: 0666, - }, - { - Name: "one/two/three/file-two", - FileMode: 0666, - }, - { - Name: "one/two/a", - FileMode: 0 | os.ModeDir | 0777, - }, - { - Name: "one/two/a/file", - FileMode: 0666, - }, - { - Name: "one/two/b", - FileMode: 0 | os.ModeDir | 0777, - }, - { - Name: "one/two/b/file", - FileMode: 0666, - }, - }, - }, - wantOutput: wantOutput{ - unix: turbopath.AnchoredUnixPathArray{ - "one", - "one/two", - "one/two/three", - "one/two/three/file-one", - "one/two/three/file-two", - "one/two/a", - "one/two/a/file", - "one/two/b", - "one/two/b/file", - }.ToSystemPathArray(), - }, - }, - { - name: "pathological cache works", - tarFiles: []tarFile{ - { - Header: &tar.Header{ - Name: "one/", - Typeflag: tar.TypeDir, - Mode: 0755, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/", - Typeflag: tar.TypeDir, - Mode: 0755, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/a/", - Typeflag: tar.TypeDir, - Mode: 0755, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/b/", - Typeflag: tar.TypeDir, - Mode: 0755, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/three/", - Typeflag: tar.TypeDir, - Mode: 0755, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/a/file", - Typeflag: tar.TypeReg, - Mode: 0644, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/b/file", - Typeflag: tar.TypeReg, - Mode: 0644, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/three/file-one", - Typeflag: tar.TypeReg, - Mode: 0644, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/three/file-two", - Typeflag: tar.TypeReg, - Mode: 0644, - }, - }, - }, - wantFiles: wantFiles{ - unix: []restoreFile{ - { - Name: "one", - FileMode: 0 | os.ModeDir | 0755, - }, - { - Name: "one/two", - FileMode: 0 | os.ModeDir | 0755, - }, - { - Name: "one/two/three", - FileMode: 0 | os.ModeDir | 0755, - }, - { - Name: "one/two/three/file-one", - FileMode: 0644, - }, - { - Name: "one/two/three/file-two", - FileMode: 0644, - }, - { - Name: "one/two/a", - FileMode: 0 | os.ModeDir | 0755, - }, - { - Name: "one/two/a/file", - FileMode: 0644, - }, - { - Name: "one/two/b", - FileMode: 0 | os.ModeDir | 0755, - }, - { - Name: "one/two/b/file", - FileMode: 0644, - }, - }, - windows: []restoreFile{ - { - Name: "one", - FileMode: 0 | os.ModeDir | 0777, - }, - { - Name: "one/two", - FileMode: 0 | os.ModeDir | 0777, - }, - { - Name: "one/two/three", - FileMode: 0 | os.ModeDir | 0777, - }, - { - Name: "one/two/three/file-one", - FileMode: 0666, - }, - { - Name: "one/two/three/file-two", - FileMode: 0666, - }, - { - Name: "one/two/a", - FileMode: 0 | os.ModeDir | 0777, - }, - { - Name: "one/two/a/file", - FileMode: 0666, - }, - { - Name: "one/two/b", - FileMode: 0 | os.ModeDir | 0777, - }, - { - Name: "one/two/b/file", - FileMode: 0666, - }, - }, - }, - wantOutput: wantOutput{ - unix: turbopath.AnchoredUnixPathArray{ - "one", - "one/two", - "one/two/a", - "one/two/b", - "one/two/three", - "one/two/a/file", - "one/two/b/file", - "one/two/three/file-one", - "one/two/three/file-two", - }.ToSystemPathArray(), - }, - }, - { - name: "hello world", - tarFiles: []tarFile{ - { - Header: &tar.Header{ - Name: "target", - Typeflag: tar.TypeReg, - Mode: 0644, - }, - Body: "target", - }, - { - Header: &tar.Header{ - Name: "source", - Linkname: "target", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - }, - wantFiles: wantFiles{ - unix: []restoreFile{ - { - Name: "source", - Linkname: "target", - FileMode: 0 | os.ModeSymlink | 0777, - }, - { - Name: "target", - FileMode: 0644, - }, - }, - windows: []restoreFile{ - { - Name: "source", - Linkname: "target", - FileMode: 0 | os.ModeSymlink | 0666, - }, - { - Name: "target", - FileMode: 0666, - }, - }, - }, - wantOutput: wantOutput{ - unix: turbopath.AnchoredUnixPathArray{"target", "source"}.ToSystemPathArray(), - }, - }, - { - name: "nested file", - tarFiles: []tarFile{ - { - Header: &tar.Header{ - Name: "folder/", - Typeflag: tar.TypeDir, - Mode: 0755, - }, - }, - { - Header: &tar.Header{ - Name: "folder/file", - Typeflag: tar.TypeReg, - Mode: 0644, - }, - Body: "file", - }, - }, - wantFiles: wantFiles{ - unix: []restoreFile{ - { - Name: "folder", - FileMode: 0 | os.ModeDir | 0755, - }, - { - Name: "folder/file", - FileMode: 0644, - }, - }, - windows: []restoreFile{ - { - Name: "folder", - FileMode: 0 | os.ModeDir | 0777, - }, - { - Name: "folder/file", - FileMode: 0666, - }, - }, - }, - wantOutput: wantOutput{ - unix: turbopath.AnchoredUnixPathArray{"folder", "folder/file"}.ToSystemPathArray(), - }, - }, - { - name: "nested symlink", - tarFiles: []tarFile{ - { - Header: &tar.Header{ - Name: "folder/", - Typeflag: tar.TypeDir, - Mode: 0755, - }, - }, - { - Header: &tar.Header{ - Name: "folder/symlink", - Linkname: "../", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - { - Header: &tar.Header{ - Name: "folder/symlink/folder-sibling", - Typeflag: tar.TypeReg, - Mode: 0644, - }, - Body: "folder-sibling", - }, - }, - wantFiles: wantFiles{ - unix: []restoreFile{ - { - Name: "folder", - FileMode: 0 | os.ModeDir | 0755, - }, - { - Name: "folder/symlink", - FileMode: 0 | os.ModeSymlink | 0777, - Linkname: "../", - }, - { - Name: "folder/symlink/folder-sibling", - FileMode: 0644, - }, - { - Name: "folder-sibling", - FileMode: 0644, - }, - }, - windows: []restoreFile{ - { - Name: "folder", - FileMode: 0 | os.ModeDir | 0777, - }, - { - Name: "folder/symlink", - FileMode: 0 | os.ModeSymlink | 0666, - Linkname: "..\\", - }, - { - Name: "folder/symlink/folder-sibling", - FileMode: 0666, - }, - { - Name: "folder-sibling", - FileMode: 0666, - }, - }, - }, - wantOutput: wantOutput{ - unix: turbopath.AnchoredUnixPathArray{"folder", "folder/symlink", "folder/symlink/folder-sibling"}.ToSystemPathArray(), - }, - }, - { - name: "pathological symlinks", - tarFiles: []tarFile{ - { - Header: &tar.Header{ - Name: "one", - Linkname: "two", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - { - Header: &tar.Header{ - Name: "two", - Linkname: "three", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - { - Header: &tar.Header{ - Name: "three", - Linkname: "real", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - { - Header: &tar.Header{ - Name: "real", - Typeflag: tar.TypeReg, - Mode: 0755, - }, - Body: "real", - }, - }, - wantFiles: wantFiles{ - unix: []restoreFile{ - { - Name: "one", - Linkname: "two", - FileMode: 0 | os.ModeSymlink | 0777, - }, - { - Name: "two", - Linkname: "three", - FileMode: 0 | os.ModeSymlink | 0777, - }, - { - Name: "three", - Linkname: "real", - FileMode: 0 | os.ModeSymlink | 0777, - }, - { - Name: "real", - FileMode: 0 | 0755, - }, - }, - windows: []restoreFile{ - { - Name: "one", - Linkname: "two", - FileMode: 0 | os.ModeSymlink | 0666, - }, - { - Name: "two", - Linkname: "three", - FileMode: 0 | os.ModeSymlink | 0666, - }, - { - Name: "three", - Linkname: "real", - FileMode: 0 | os.ModeSymlink | 0666, - }, - { - Name: "real", - FileMode: 0 | 0666, - }, - }, - }, - wantOutput: wantOutput{ - unix: turbopath.AnchoredUnixPathArray{"real", "three", "two", "one"}.ToSystemPathArray(), - }, - }, - { - name: "place file at dir location", - tarFiles: []tarFile{ - { - Header: &tar.Header{ - Name: "folder-not-file/", - Typeflag: tar.TypeDir, - Mode: 0755, - }, - }, - { - Header: &tar.Header{ - Name: "folder-not-file/subfile", - Typeflag: tar.TypeReg, - Mode: 0755, - }, - Body: "subfile", - }, - { - Header: &tar.Header{ - Name: "folder-not-file", - Typeflag: tar.TypeReg, - Mode: 0755, - }, - Body: "this shouldn't work", - }, - }, - wantFiles: wantFiles{ - unix: []restoreFile{ - { - Name: "folder-not-file", - FileMode: 0 | os.ModeDir | 0755, - }, - { - Name: "folder-not-file/subfile", - FileMode: 0755, - }, - }, - windows: []restoreFile{ - { - Name: "folder-not-file", - FileMode: 0 | os.ModeDir | 0777, - }, - { - Name: "folder-not-file/subfile", - FileMode: 0666, - }, - }, - }, - wantOutput: wantOutput{ - unix: turbopath.AnchoredUnixPathArray{"folder-not-file", "folder-not-file/subfile"}.ToSystemPathArray(), - }, - wantErr: wantErr{ - unix: syscall.EISDIR, - windows: syscall.EISDIR, - }, - }, - // { - // name: "missing symlink with file at subdir", - // tarFiles: []tarFile{ - // { - // Header: &tar.Header{ - // Name: "one", - // Linkname: "two", - // Typeflag: tar.TypeSymlink, - // Mode: 0777, - // }, - // }, - // { - // Header: &tar.Header{ - // Name: "one/file", - // Typeflag: tar.TypeReg, - // Mode: 0755, - // }, - // Body: "file", - // }, - // }, - // wantFiles: wantFiles{ - // unix: []restoreFile{ - // { - // Name: "one", - // Linkname: "two", - // FileMode: 0 | os.ModeSymlink | 0777, - // }, - // }, - // }, - // wantOutput: wantOutput{ - // unix: turbopath.AnchoredUnixPathArray{"one"}.ToSystemPathArray(), - // windows: nil, - // }, - // wantErr: wantErr{ - // unix: os.ErrExist, - // windows: os.ErrExist, - // }, - // }, - { - name: "symlink cycle", - tarFiles: []tarFile{ - { - Header: &tar.Header{ - Name: "one", - Linkname: "two", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - { - Header: &tar.Header{ - Name: "two", - Linkname: "three", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - { - Header: &tar.Header{ - Name: "three", - Linkname: "one", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - }, - wantFiles: wantFiles{ - unix: []restoreFile{}, - }, - wantOutput: wantOutput{ - unix: []turbopath.AnchoredSystemPath{}, - }, - wantErr: wantErr{ - unix: errCycleDetected, - windows: errCycleDetected, - }, - }, - { - name: "symlink clobber", - tarFiles: []tarFile{ - { - Header: &tar.Header{ - Name: "one", - Linkname: "two", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - { - Header: &tar.Header{ - Name: "one", - Linkname: "three", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - { - Header: &tar.Header{ - Name: "one", - Linkname: "real", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - { - Header: &tar.Header{ - Name: "real", - Typeflag: tar.TypeReg, - Mode: 0755, - }, - Body: "real", - }, - }, - wantFiles: wantFiles{ - unix: []restoreFile{ - { - Name: "one", - Linkname: "real", - FileMode: 0 | os.ModeSymlink | 0777, - }, - { - Name: "real", - FileMode: 0755, - }, - }, - windows: []restoreFile{ - { - Name: "one", - Linkname: "real", - FileMode: 0 | os.ModeSymlink | 0666, - }, - { - Name: "real", - FileMode: 0666, - }, - }, - }, - wantOutput: wantOutput{ - unix: turbopath.AnchoredUnixPathArray{"real", "one"}.ToSystemPathArray(), - }, - }, - { - name: "symlink traversal", - tarFiles: []tarFile{ - { - Header: &tar.Header{ - Name: "escape", - Linkname: "../", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - { - Header: &tar.Header{ - Name: "escape/file", - Typeflag: tar.TypeReg, - Mode: 0644, - }, - Body: "file", - }, - }, - wantFiles: wantFiles{ - unix: []restoreFile{ - { - Name: "escape", - Linkname: "../", - FileMode: 0 | os.ModeSymlink | 0777, - }, - }, - windows: []restoreFile{ - { - Name: "escape", - Linkname: "..\\", - FileMode: 0 | os.ModeSymlink | 0666, - }, - }, - }, - wantOutput: wantOutput{ - unix: turbopath.AnchoredUnixPathArray{"escape"}.ToSystemPathArray(), - }, - wantErr: wantErr{ - unix: errTraversal, - windows: errTraversal, - }, - }, - { - name: "Double indirection: file", - tarFiles: []tarFile{ - { - Header: &tar.Header{ - Name: "up", - Linkname: "../", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - { - Header: &tar.Header{ - Name: "link", - Linkname: "up", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - { - Header: &tar.Header{ - Name: "link/outside-file", - Typeflag: tar.TypeReg, - Mode: 0755, - }, - }, - }, - wantErr: wantErr{unix: errTraversal, windows: errTraversal}, - wantOutput: wantOutput{ - unix: turbopath.AnchoredUnixPathArray{ - "up", - "link", - }.ToSystemPathArray(), - }, - }, - { - name: "Double indirection: folder", - tarFiles: []tarFile{ - { - Header: &tar.Header{ - Name: "up", - Linkname: "../", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - { - Header: &tar.Header{ - Name: "link", - Linkname: "up", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - { - Header: &tar.Header{ - Name: "link/level-one/level-two/", - Typeflag: tar.TypeDir, - Mode: 0755, - }, - }, - }, - wantErr: wantErr{unix: errTraversal, windows: errTraversal}, - wantOutput: wantOutput{ - unix: turbopath.AnchoredUnixPathArray{ - "up", - "link", - }.ToSystemPathArray(), - }, - }, - { - name: "name traversal", - tarFiles: []tarFile{ - { - Header: &tar.Header{ - Name: "../escape", - Typeflag: tar.TypeReg, - Mode: 0644, - }, - Body: "file", - }, - }, - wantFiles: wantFiles{ - unix: []restoreFile{}, - }, - wantOutput: wantOutput{ - unix: []turbopath.AnchoredSystemPath{}, - }, - wantErr: wantErr{ - unix: errNameMalformed, - windows: errNameMalformed, - }, - }, - { - name: "windows unsafe", - tarFiles: []tarFile{ - { - Header: &tar.Header{ - Name: "back\\slash\\file", - Typeflag: tar.TypeReg, - Mode: 0644, - }, - Body: "file", - }, - }, - wantFiles: wantFiles{ - unix: []restoreFile{ - { - Name: "back\\slash\\file", - FileMode: 0644, - }, - }, - windows: []restoreFile{}, - }, - wantOutput: wantOutput{ - unix: turbopath.AnchoredUnixPathArray{"back\\slash\\file"}.ToSystemPathArray(), - windows: turbopath.AnchoredUnixPathArray{}.ToSystemPathArray(), - }, - wantErr: wantErr{ - unix: nil, - windows: errNameWindowsUnsafe, - }, - }, - { - name: "fifo (and others) unsupported", - tarFiles: []tarFile{ - { - Header: &tar.Header{ - Name: "fifo", - Typeflag: tar.TypeFifo, - }, - }, - }, - wantFiles: wantFiles{ - unix: []restoreFile{}, - }, - wantOutput: wantOutput{ - unix: []turbopath.AnchoredSystemPath{}, - }, - wantErr: wantErr{ - unix: errUnsupportedFileType, - windows: errUnsupportedFileType, - }, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - archivePath := generateTar(t, tt.tarFiles) - anchor := generateAnchor(t) - - cacheItem, err := Open(archivePath) - assert.NilError(t, err, "Open") - - restoreOutput, restoreErr := cacheItem.Restore(anchor) - var desiredErr error - if runtime.GOOS == "windows" { - desiredErr = tt.wantErr.windows - } else { - desiredErr = tt.wantErr.unix - } - if desiredErr != nil { - if !errors.Is(restoreErr, desiredErr) { - t.Errorf("wanted err: %v, got err: %v", tt.wantErr, restoreErr) - } - } else { - assert.NilError(t, restoreErr, "Restore") - } - - outputComparison := tt.wantOutput.unix - if runtime.GOOS == "windows" && tt.wantOutput.windows != nil { - outputComparison = tt.wantOutput.windows - } - - if !reflect.DeepEqual(restoreOutput, outputComparison) { - t.Errorf("Restore() = %v, want %v", restoreOutput, outputComparison) - } - - // Check files on disk. - filesComparison := tt.wantFiles.unix - if runtime.GOOS == "windows" && tt.wantFiles.windows != nil { - filesComparison = tt.wantFiles.windows - } - for _, diskFile := range filesComparison { - assertFileExists(t, anchor, diskFile) - } - - assert.NilError(t, cacheItem.Close(), "Close") - }) - } -} - -func Test_checkName(t *testing.T) { - tests := []struct { - path string - wellFormed bool - windowsSafe bool - }{ - // Empty - { - path: "", - wellFormed: false, - windowsSafe: false, - }, - // Bad prefix - { - path: ".", - wellFormed: false, - windowsSafe: true, - }, - { - path: "..", - wellFormed: false, - windowsSafe: true, - }, - { - path: "/", - wellFormed: false, - windowsSafe: true, - }, - { - path: "./", - wellFormed: false, - windowsSafe: true, - }, - { - path: "../", - wellFormed: false, - windowsSafe: true, - }, - // Bad prefix, suffixed - { - path: "/a", - wellFormed: false, - windowsSafe: true, - }, - { - path: "./a", - wellFormed: false, - windowsSafe: true, - }, - { - path: "../a", - wellFormed: false, - windowsSafe: true, - }, - // Bad Suffix - { - path: "/.", - wellFormed: false, - windowsSafe: true, - }, - { - path: "/..", - wellFormed: false, - windowsSafe: true, - }, - // Bad Suffix, with prefix - { - path: "a/.", - wellFormed: false, - windowsSafe: true, - }, - { - path: "a/..", - wellFormed: false, - windowsSafe: true, - }, - // Bad middle - { - path: "//", - wellFormed: false, - windowsSafe: true, - }, - { - path: "/./", - wellFormed: false, - windowsSafe: true, - }, - { - path: "/../", - wellFormed: false, - windowsSafe: true, - }, - // Bad middle, prefixed - { - path: "a//", - wellFormed: false, - windowsSafe: true, - }, - { - path: "a/./", - wellFormed: false, - windowsSafe: true, - }, - { - path: "a/../", - wellFormed: false, - windowsSafe: true, - }, - // Bad middle, suffixed - { - path: "//a", - wellFormed: false, - windowsSafe: true, - }, - { - path: "/./a", - wellFormed: false, - windowsSafe: true, - }, - { - path: "/../a", - wellFormed: false, - windowsSafe: true, - }, - // Bad middle, wrapped - { - path: "a//a", - wellFormed: false, - windowsSafe: true, - }, - { - path: "a/./a", - wellFormed: false, - windowsSafe: true, - }, - { - path: "a/../a", - wellFormed: false, - windowsSafe: true, - }, - // False positive tests - { - path: "...", - wellFormed: true, - windowsSafe: true, - }, - { - path: ".../a", - wellFormed: true, - windowsSafe: true, - }, - { - path: "a/...", - wellFormed: true, - windowsSafe: true, - }, - { - path: "a/.../a", - wellFormed: true, - windowsSafe: true, - }, - { - path: ".../...", - wellFormed: true, - windowsSafe: true, - }, - } - for _, tt := range tests { - t.Run(fmt.Sprintf("Path: \"%v\"", tt.path), func(t *testing.T) { - wellFormed, windowsSafe := checkName(tt.path) - if wellFormed != tt.wellFormed || windowsSafe != tt.windowsSafe { - t.Errorf("\nwantOutput: checkName(\"%v\") wellFormed = %v, windowsSafe %v\ngot: checkName(\"%v\") wellFormed = %v, windowsSafe %v", tt.path, tt.wellFormed, tt.windowsSafe, tt.path, wellFormed, windowsSafe) - } - }) - } -} - -func Test_canonicalizeLinkname(t *testing.T) { - // We're lying that this thing is absolute, but that's not relevant for tests. - anchor := turbopath.AbsoluteSystemPath(filepath.Join("path", "to", "anchor")) - - tests := []struct { - name string - processedName turbopath.AnchoredSystemPath - linkname string - canonicalUnix string - canonicalWindows string - }{ - { - name: "hello world", - processedName: turbopath.AnchoredSystemPath("source"), - linkname: "target", - canonicalUnix: "path/to/anchor/target", - canonicalWindows: "path\\to\\anchor\\target", - }, - { - name: "Unix path subdirectory traversal", - processedName: turbopath.AnchoredUnixPath("child/source").ToSystemPath(), - linkname: "../sibling/target", - canonicalUnix: "path/to/anchor/sibling/target", - canonicalWindows: "path\\to\\anchor\\sibling\\target", - }, - { - name: "Windows path subdirectory traversal", - processedName: turbopath.AnchoredUnixPath("child/source").ToSystemPath(), - linkname: "..\\sibling\\target", - canonicalUnix: "path/to/anchor/child/..\\sibling\\target", - canonicalWindows: "path\\to\\anchor\\sibling\\target", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - canonical := tt.canonicalUnix - if runtime.GOOS == "windows" { - canonical = tt.canonicalWindows - } - if got := canonicalizeLinkname(anchor, tt.processedName, tt.linkname); got != canonical { - t.Errorf("canonicalizeLinkname() = %v, want %v", got, canonical) - } - }) - } -} - -func Test_canonicalizeName(t *testing.T) { - tests := []struct { - name string - fileName string - want turbopath.AnchoredSystemPath - wantErr error - }{ - { - name: "hello world", - fileName: "test.txt", - want: "test.txt", - }, - { - name: "directory", - fileName: "something/", - want: "something", - }, - { - name: "malformed name", - fileName: "//", - want: "", - wantErr: errNameMalformed, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - got, err := canonicalizeName(tt.fileName) - if tt.wantErr != nil && !errors.Is(err, tt.wantErr) { - t.Errorf("canonicalizeName() error = %v, wantErr %v", err, tt.wantErr) - return - } - if !reflect.DeepEqual(got, tt.want) { - t.Errorf("canonicalizeName() = %v, want %v", got, tt.want) - } - }) - } -} - -func TestCacheItem_Restore(t *testing.T) { - tests := []struct { - name string - tarFiles []tarFile - want []turbopath.AnchoredSystemPath - }{ - { - name: "duplicate restores", - tarFiles: []tarFile{ - { - Header: &tar.Header{ - Name: "target", - Typeflag: tar.TypeReg, - Mode: 0644, - }, - Body: "target", - }, - { - Header: &tar.Header{ - Name: "source", - Linkname: "target", - Typeflag: tar.TypeSymlink, - Mode: 0777, - }, - }, - { - Header: &tar.Header{ - Name: "one/", - Typeflag: tar.TypeDir, - Mode: 0755, - }, - }, - { - Header: &tar.Header{ - Name: "one/two/", - Typeflag: tar.TypeDir, - Mode: 0755, - }, - }, - }, - want: turbopath.AnchoredUnixPathArray{"target", "source", "one", "one/two"}.ToSystemPathArray(), - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - archivePath := generateTar(t, tt.tarFiles) - anchor := generateAnchor(t) - - cacheItem, err := Open(archivePath) - assert.NilError(t, err, "Open") - - restoreOutput, restoreErr := cacheItem.Restore(anchor) - if !reflect.DeepEqual(restoreOutput, tt.want) { - t.Errorf("#1 CacheItem.Restore() = %v, want %v", restoreOutput, tt.want) - } - assert.NilError(t, restoreErr, "Restore #1") - assert.NilError(t, cacheItem.Close(), "Close") - - cacheItem2, err2 := Open(archivePath) - assert.NilError(t, err2, "Open") - - restoreOutput2, restoreErr2 := cacheItem2.Restore(anchor) - if !reflect.DeepEqual(restoreOutput2, tt.want) { - t.Errorf("#2 CacheItem.Restore() = %v, want %v", restoreOutput2, tt.want) - } - assert.NilError(t, restoreErr2, "Restore #2") - assert.NilError(t, cacheItem2.Close(), "Close") - }) - } -} diff --git a/cli/internal/filewatcher/cookie.go b/cli/internal/filewatcher/cookie.go index ad039c065fed0..d76aa45064445 100644 --- a/cli/internal/filewatcher/cookie.go +++ b/cli/internal/filewatcher/cookie.go @@ -41,7 +41,7 @@ func NewCookieJar(cookieDir turbopath.AbsoluteSystemPath, timeout time.Duration) if err := cookieDir.RemoveAll(); err != nil { return nil, err } - if err := cookieDir.MkdirAll(0775); err != nil { + if err := cookieDir.MkdirAll(); err != nil { return nil, err } return &CookieJar{ diff --git a/cli/internal/filewatcher/filewatcher_test.go b/cli/internal/filewatcher/filewatcher_test.go index 81ab0177da243..a08c0070e59b8 100644 --- a/cli/internal/filewatcher/filewatcher_test.go +++ b/cli/internal/filewatcher/filewatcher_test.go @@ -83,13 +83,13 @@ func TestFileWatching(t *testing.T) { logger := hclog.Default() logger.SetLevel(hclog.Debug) repoRoot := fs.AbsoluteSystemPathFromUpstream(t.TempDir()) - err := repoRoot.UntypedJoin(".git").MkdirAll(0775) + err := repoRoot.UntypedJoin(".git").MkdirAll() assert.NilError(t, err, "MkdirAll") - err = repoRoot.UntypedJoin("node_modules", "some-dep").MkdirAll(0775) + err = repoRoot.UntypedJoin("node_modules", "some-dep").MkdirAll() assert.NilError(t, err, "MkdirAll") - err = repoRoot.UntypedJoin("parent", "child").MkdirAll(0775) + err = repoRoot.UntypedJoin("parent", "child").MkdirAll() assert.NilError(t, err, "MkdirAll") - err = repoRoot.UntypedJoin("parent", "sibling").MkdirAll(0775) + err = repoRoot.UntypedJoin("parent", "sibling").MkdirAll() assert.NilError(t, err, "MkdirAll") // Directory layout: @@ -130,7 +130,7 @@ func TestFileWatching(t *testing.T) { }) deepPath := repoRoot.UntypedJoin("parent", "sibling", "deep", "path") - err = deepPath.MkdirAll(0775) + err = deepPath.MkdirAll() assert.NilError(t, err, "MkdirAll") // We'll catch an event for "deep", but not "deep/path" since // we don't have a recursive watch diff --git a/cli/internal/globby/globby.go b/cli/internal/globby/globby.go index 41d1905ed0297..7b107586cabac 100644 --- a/cli/internal/globby/globby.go +++ b/cli/internal/globby/globby.go @@ -3,7 +3,6 @@ package globby import ( "fmt" "path/filepath" - "sort" "strings" iofs "io/fs" @@ -14,30 +13,12 @@ import ( "github.com/vercel/turborepo/cli/internal/util" ) -// GlobAll returns an array of files and folders that match the specified set of glob patterns. -// The returned files and folders are absolute paths, assuming that basePath is an absolute path. -func GlobAll(basePath string, includePatterns []string, excludePatterns []string) ([]string, error) { - fsys := fs.CreateDirFSAtRoot(basePath) - fsysRoot := fs.GetDirFSRootPath(fsys) - output, err := globAllFs(fsys, fsysRoot, basePath, includePatterns, excludePatterns) - - // Because this is coming out of a map output is in no way ordered. - // Sorting will put the files in a depth-first order. - sort.Strings(output) - return output, err -} - // GlobFiles returns an array of files that match the specified set of glob patterns. // The return files are absolute paths, assuming that basePath is an absolute path. func GlobFiles(basePath string, includePatterns []string, excludePatterns []string) ([]string, error) { fsys := fs.CreateDirFSAtRoot(basePath) fsysRoot := fs.GetDirFSRootPath(fsys) - output, err := globFilesFs(fsys, fsysRoot, basePath, includePatterns, excludePatterns) - - // Because this is coming out of a map output is in no way ordered. - // Sorting will put the files in a depth-first order. - sort.Strings(output) - return output, err + return globFilesFs(fsys, fsysRoot, basePath, includePatterns, excludePatterns) } // checkRelativePath ensures that the the requested file path is a child of `from`. @@ -55,18 +36,8 @@ func checkRelativePath(from string, to string) error { return nil } -// globFilesFs searches the specified file system to enumerate all files to include. +// globFilesFs searches the specified file system to ensure to enumerate all files to include. func globFilesFs(fsys iofs.FS, fsysRoot string, basePath string, includePatterns []string, excludePatterns []string) ([]string, error) { - return globWalkFs(fsys, fsysRoot, basePath, includePatterns, excludePatterns, false) -} - -// globAllFs searches the specified file system to enumerate all files to include. -func globAllFs(fsys iofs.FS, fsysRoot string, basePath string, includePatterns []string, excludePatterns []string) ([]string, error) { - return globWalkFs(fsys, fsysRoot, basePath, includePatterns, excludePatterns, true) -} - -// globWalkFs searches the specified file system to enumerate all files and folders to include. -func globWalkFs(fsys iofs.FS, fsysRoot string, basePath string, includePatterns []string, excludePatterns []string, includeDirs bool) ([]string, error) { var processedIncludes []string var processedExcludes []string result := make(util.Set) @@ -134,7 +105,7 @@ func globWalkFs(fsys iofs.FS, fsysRoot string, basePath string, includePatterns excludePattern = filepath.ToSlash(excludePattern) err := doublestar.GlobWalk(fsys, includePattern, func(path string, dirEntry iofs.DirEntry) error { - if !includeDirs && dirEntry.IsDir() { + if dirEntry.IsDir() { return nil } @@ -173,9 +144,5 @@ func globWalkFs(fsys iofs.FS, fsysRoot string, basePath string, includePatterns return nil, err } - // Never actually capture the root folder. - // This is a risk because of how we rework the globs. - result.Delete(strings.TrimSuffix(basePath, "/")) - return result.UnsafeListOfStrings(), nil } diff --git a/cli/internal/globby/globby_test.go b/cli/internal/globby/globby_test.go index 922b1dc4b587d..04dff9b7e23ae 100644 --- a/cli/internal/globby/globby_test.go +++ b/cli/internal/globby/globby_test.go @@ -31,12 +31,11 @@ func TestGlobFilesFs(t *testing.T) { excludePatterns []string } tests := []struct { - name string - files []string - args args - wantAll []string - wantFiles []string - wantErr bool + name string + files []string + args args + want []string + wantErr bool }{ { name: "hello world", @@ -46,29 +45,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"*.txt"}, excludePatterns: []string{}, }, - wantAll: []string{"/test.txt"}, - wantFiles: []string{"/test.txt"}, - }, - { - name: "bullet files", - files: []string{ - "/test.txt", - "/subdir/test.txt", - "/other/test.txt", - }, - args: args{ - basePath: "/", - includePatterns: []string{"subdir/test.txt", "test.txt"}, - excludePatterns: []string{}, - }, - wantAll: []string{ - "/subdir/test.txt", - "/test.txt", - }, - wantFiles: []string{ - "/subdir/test.txt", - "/test.txt", - }, + want: []string{"/test.txt"}, }, { name: "finding workspace package.json files", @@ -92,14 +69,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"packages/*/package.json", "apps/*/package.json"}, excludePatterns: []string{"**/node_modules/", "**/bower_components/", "**/test/", "**/tests/"}, }, - wantAll: []string{ - "/repos/some-app/apps/docs/package.json", - "/repos/some-app/apps/web/package.json", - "/repos/some-app/packages/colors/package.json", - "/repos/some-app/packages/faker/package.json", - "/repos/some-app/packages/left-pad/package.json", - }, - wantFiles: []string{ + want: []string{ "/repos/some-app/apps/docs/package.json", "/repos/some-app/apps/web/package.json", "/repos/some-app/packages/colors/package.json", @@ -129,16 +99,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"**/package.json"}, excludePatterns: []string{"**/node_modules/", "**/bower_components/", "**/test/", "**/tests/"}, }, - wantAll: []string{ - "/repos/some-app/apps/docs/package.json", - "/repos/some-app/apps/web/package.json", - "/repos/some-app/examples/package.json", - "/repos/some-app/package.json", - "/repos/some-app/packages/colors/package.json", - "/repos/some-app/packages/faker/package.json", - "/repos/some-app/packages/left-pad/package.json", - }, - wantFiles: []string{ + want: []string{ "/repos/some-app/apps/docs/package.json", "/repos/some-app/apps/web/package.json", "/repos/some-app/examples/package.json", @@ -176,14 +137,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"packages/**/package.json"}, excludePatterns: []string{"**/node_modules/", "**/bower_components/", "**/test/", "**/tests/"}, }, - wantAll: []string{ - "/repos/some-app/packages/colors/package.json", - "/repos/some-app/packages/faker/package.json", - "/repos/some-app/packages/left-pad/package.json", - "/repos/some-app/packages/xzibit/package.json", - "/repos/some-app/packages/xzibit/packages/yo-dawg/package.json", - }, - wantFiles: []string{ + want: []string{ "/repos/some-app/packages/colors/package.json", "/repos/some-app/packages/faker/package.json", "/repos/some-app/packages/left-pad/package.json", @@ -219,14 +173,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"packages/**/package.json", "tests/mocks/*/package.json"}, excludePatterns: []string{"**/node_modules/", "**/bower_components/", "**/test/", "**/tests/"}, }, - wantAll: []string{ - "/repos/some-app/packages/colors/package.json", - "/repos/some-app/packages/faker/package.json", - "/repos/some-app/packages/left-pad/package.json", - "/repos/some-app/packages/xzibit/package.json", - "/repos/some-app/packages/xzibit/packages/yo-dawg/package.json", - }, - wantFiles: []string{ + want: []string{ "/repos/some-app/packages/colors/package.json", "/repos/some-app/packages/faker/package.json", "/repos/some-app/packages/left-pad/package.json", @@ -256,26 +203,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{".turbo/turbo-build.log", "dist/**", ".next/**", "public/dist/**"}, excludePatterns: []string{}, }, - wantAll: []string{ - "/repos/some-app/.next", - "/repos/some-app/.next/cache", - "/repos/some-app/.next/cache/db6a76a62043520e7aaadd0bb2104e78.txt", - "/repos/some-app/.next/log.txt", - "/repos/some-app/.turbo/turbo-build.log", - "/repos/some-app/dist", - "/repos/some-app/dist/index.html", - "/repos/some-app/dist/js", - "/repos/some-app/dist/js/index.js", - "/repos/some-app/dist/js/lib.js", - "/repos/some-app/dist/js/node_modules", - "/repos/some-app/dist/js/node_modules/browserify.js", - "/repos/some-app/public/dist", - "/repos/some-app/public/dist/css", - "/repos/some-app/public/dist/css/index.css", - "/repos/some-app/public/dist/images", - "/repos/some-app/public/dist/images/rick_astley.jpg", - }, - wantFiles: []string{ + want: []string{ "/repos/some-app/.next/cache/db6a76a62043520e7aaadd0bb2104e78.txt", "/repos/some-app/.next/log.txt", "/repos/some-app/.turbo/turbo-build.log", @@ -300,16 +228,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"dist/**"}, excludePatterns: []string{}, }, - wantAll: []string{ - "/repos/some-app/dist", - "/repos/some-app/dist/index.html", - "/repos/some-app/dist/js", - "/repos/some-app/dist/js/index.js", - "/repos/some-app/dist/js/lib.js", - "/repos/some-app/dist/js/node_modules", - "/repos/some-app/dist/js/node_modules/browserify.js", - }, - wantFiles: []string{ + want: []string{ "/repos/some-app/dist/index.html", "/repos/some-app/dist/js/index.js", "/repos/some-app/dist/js/lib.js", @@ -329,8 +248,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"dist"}, excludePatterns: []string{}, }, - wantAll: []string{"/repos/some-app/dist"}, - wantFiles: []string{}, + want: []string{}, }, { name: "redundant includes do not duplicate", @@ -345,16 +263,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"**/*", "dist/**"}, excludePatterns: []string{}, }, - wantAll: []string{ - "/repos/some-app/dist", - "/repos/some-app/dist/index.html", - "/repos/some-app/dist/js", - "/repos/some-app/dist/js/index.js", - "/repos/some-app/dist/js/lib.js", - "/repos/some-app/dist/js/node_modules", - "/repos/some-app/dist/js/node_modules/browserify.js", - }, - wantFiles: []string{ + want: []string{ "/repos/some-app/dist/index.html", "/repos/some-app/dist/js/index.js", "/repos/some-app/dist/js/lib.js", @@ -374,8 +283,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"**"}, excludePatterns: []string{"**"}, }, - wantAll: []string{}, - wantFiles: []string{}, + want: []string{}, }, { name: "passing just a directory to exclude prevents capture of children", @@ -390,11 +298,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"dist/**"}, excludePatterns: []string{"dist/js"}, }, - wantAll: []string{ - "/repos/some-app/dist", - "/repos/some-app/dist/index.html", - }, - wantFiles: []string{ + want: []string{ "/repos/some-app/dist/index.html", }, }, @@ -411,12 +315,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"dist/**"}, excludePatterns: []string{"dist/js/**"}, }, - wantAll: []string{ - "/repos/some-app/dist", - "/repos/some-app/dist/index.html", - "/repos/some-app/dist/js", - }, - wantFiles: []string{ + want: []string{ "/repos/some-app/dist/index.html", }, }, @@ -433,8 +332,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"**"}, excludePatterns: []string{"./"}, }, - wantAll: []string{}, - wantFiles: []string{}, + want: []string{}, }, { name: "exclude everything with traversal applies at a non-base path", @@ -449,8 +347,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"**"}, excludePatterns: []string{"./dist"}, }, - wantAll: []string{}, - wantFiles: []string{}, + want: []string{}, }, { name: "exclude everything with folder traversal (..) applies at base path", @@ -465,8 +362,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"**"}, excludePatterns: []string{"dist/../"}, }, - wantAll: []string{}, - wantFiles: []string{}, + want: []string{}, }, { name: "how do globs even work bad glob microformat", @@ -481,16 +377,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"**/**/**"}, excludePatterns: []string{}, }, - wantAll: []string{ - "/repos/some-app/dist", - "/repos/some-app/dist/index.html", - "/repos/some-app/dist/js", - "/repos/some-app/dist/js/index.js", - "/repos/some-app/dist/js/lib.js", - "/repos/some-app/dist/js/node_modules", - "/repos/some-app/dist/js/node_modules/browserify.js", - }, - wantFiles: []string{ + want: []string{ "/repos/some-app/dist/index.html", "/repos/some-app/dist/js/index.js", "/repos/some-app/dist/js/lib.js", @@ -511,9 +398,8 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"../spanish-inquisition/**", "dist/**"}, excludePatterns: []string{}, }, - wantAll: []string{}, - wantFiles: []string{}, - wantErr: true, + want: []string{}, + wantErr: true, }, { name: "globs and traversal and globs do not cross base path", @@ -529,9 +415,8 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"**/../../spanish-inquisition/**"}, excludePatterns: []string{}, }, - wantAll: []string{}, - wantFiles: []string{}, - wantErr: true, + want: []string{}, + wantErr: true, }, { name: "traversal works within base path", @@ -546,16 +431,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"dist/js/../**"}, excludePatterns: []string{}, }, - wantAll: []string{ - "/repos/some-app/dist", - "/repos/some-app/dist/index.html", - "/repos/some-app/dist/js", - "/repos/some-app/dist/js/index.js", - "/repos/some-app/dist/js/lib.js", - "/repos/some-app/dist/js/node_modules", - "/repos/some-app/dist/js/node_modules/browserify.js", - }, - wantFiles: []string{ + want: []string{ "/repos/some-app/dist/index.html", "/repos/some-app/dist/js/index.js", "/repos/some-app/dist/js/lib.js", @@ -575,16 +451,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"dist/./././**"}, excludePatterns: []string{}, }, - wantAll: []string{ - "/repos/some-app/dist", - "/repos/some-app/dist/index.html", - "/repos/some-app/dist/js", - "/repos/some-app/dist/js/index.js", - "/repos/some-app/dist/js/lib.js", - "/repos/some-app/dist/js/node_modules", - "/repos/some-app/dist/js/node_modules/browserify.js", - }, - wantFiles: []string{ + want: []string{ "/repos/some-app/dist/index.html", "/repos/some-app/dist/js/index.js", "/repos/some-app/dist/js/lib.js", @@ -592,7 +459,7 @@ func TestGlobFilesFs(t *testing.T) { }, }, { - name: "depth of 1 includes handles folders properly", + name: "depth of 1 includes does not capture folders", files: []string{ "/repos/some-app/package.json", "/repos/some-app/dist/index.html", @@ -605,11 +472,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"*"}, excludePatterns: []string{}, }, - wantAll: []string{ - "/repos/some-app/dist", - "/repos/some-app/package.json", - }, - wantFiles: []string{"/repos/some-app/package.json"}, + want: []string{"/repos/some-app/package.json"}, }, { name: "depth of 1 excludes prevents capturing folders", @@ -625,11 +488,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"**"}, excludePatterns: []string{"dist/*"}, }, - wantAll: []string{ - "/repos/some-app/dist", - "/repos/some-app/package.json", - }, - wantFiles: []string{"/repos/some-app/package.json"}, + want: []string{"/repos/some-app/package.json"}, }, { name: "No-trailing slash basePath works", @@ -644,16 +503,7 @@ func TestGlobFilesFs(t *testing.T) { includePatterns: []string{"dist/**"}, excludePatterns: []string{}, }, - wantAll: []string{ - "/repos/some-app/dist", - "/repos/some-app/dist/index.html", - "/repos/some-app/dist/js", - "/repos/some-app/dist/js/index.js", - "/repos/some-app/dist/js/lib.js", - "/repos/some-app/dist/js/node_modules", - "/repos/some-app/dist/js/node_modules/browserify.js", - }, - wantFiles: []string{ + want: []string{ "/repos/some-app/dist/index.html", "/repos/some-app/dist/js/index.js", "/repos/some-app/dist/js/lib.js", @@ -680,28 +530,8 @@ func TestGlobFilesFs(t *testing.T) { sort.Strings(gotToSlash) - if !reflect.DeepEqual(gotToSlash, tt.wantFiles) { - t.Errorf("globFilesFs() = %v, want %v", gotToSlash, tt.wantFiles) - } - }) - - t.Run(tt.name, func(t *testing.T) { - got, err := globAllFs(fsys, fsysRoot, tt.args.basePath, tt.args.includePatterns, tt.args.excludePatterns) - - if (err != nil) != tt.wantErr { - t.Errorf("globAllFs() error = %v, wantErr %v", err, tt.wantErr) - return - } - - gotToSlash := make([]string, len(got)) - for index, path := range got { - gotToSlash[index] = filepath.ToSlash(path) - } - - sort.Strings(gotToSlash) - - if !reflect.DeepEqual(gotToSlash, tt.wantAll) { - t.Errorf("globAllFs() = %v, want %v", gotToSlash, tt.wantAll) + if !reflect.DeepEqual(gotToSlash, tt.want) { + t.Errorf("globFilesFs() = %v, want %v", gotToSlash, tt.want) } }) } diff --git a/cli/internal/hashing/package_deps_hash_test.go b/cli/internal/hashing/package_deps_hash_test.go index 9fd5321115a60..472d586b3b00a 100644 --- a/cli/internal/hashing/package_deps_hash_test.go +++ b/cli/internal/hashing/package_deps_hash_test.go @@ -242,7 +242,7 @@ func TestGetPackageDeps(t *testing.T) { myPkgDir := repoRoot.UntypedJoin("my-pkg") // create the dir first - err := myPkgDir.MkdirAll(0775) + err := myPkgDir.MkdirAll() assert.NilError(t, err, "CreateDir") // create file 1 diff --git a/cli/internal/runcache/runcache.go b/cli/internal/runcache/runcache.go index dc1b4433ac46e..99ef25c25bbd1 100644 --- a/cli/internal/runcache/runcache.go +++ b/cli/internal/runcache/runcache.go @@ -273,7 +273,7 @@ func (tc TaskCache) SaveOutputs(ctx context.Context, logger hclog.Logger, termin logger.Debug("caching output", "outputs", tc.repoRelativeGlobs) - filesToBeCached, err := globby.GlobAll(tc.rc.repoRoot.ToStringDuringMigration(), tc.repoRelativeGlobs.Inclusions, tc.repoRelativeGlobs.Exclusions) + filesToBeCached, err := globby.GlobFiles(tc.rc.repoRoot.ToStringDuringMigration(), tc.repoRelativeGlobs.Inclusions, tc.repoRelativeGlobs.Exclusions) if err != nil { return err } diff --git a/cli/internal/turbopath/absolute_system_path.go b/cli/internal/turbopath/absolute_system_path.go index 033e24b158ae9..ba10f918eb194 100644 --- a/cli/internal/turbopath/absolute_system_path.go +++ b/cli/internal/turbopath/absolute_system_path.go @@ -51,14 +51,9 @@ func (p AbsoluteSystemPath) Dir() AbsoluteSystemPath { return AbsoluteSystemPath(filepath.Dir(p.ToString())) } -// Mkdir implements os.Mkdir(p, perm) -func (p AbsoluteSystemPath) Mkdir(perm os.FileMode) error { - return os.Mkdir(p.ToString(), perm) -} - -// MkdirAll implements os.MkdirAll(p, perm) -func (p AbsoluteSystemPath) MkdirAll(perm os.FileMode) error { - return os.MkdirAll(p.ToString(), perm) +// MkdirAll implements os.MkdirAll(p, DirPermissions|0644) +func (p AbsoluteSystemPath) MkdirAll() error { + return os.MkdirAll(p.ToString(), _dirPermissions|0644) } // Open implements os.Open(p) for an AbsoluteSystemPath @@ -218,23 +213,3 @@ func (p AbsoluteSystemPath) EvalSymlinks() (AbsoluteSystemPath, error) { } return AbsoluteSystemPath(result), nil } - -// HasPrefix is strings.HasPrefix for paths, ensuring that it matches on separator boundaries. -// This does NOT perform Clean in advance. -func (p AbsoluteSystemPath) HasPrefix(prefix AbsoluteSystemPath) bool { - prefixLen := len(prefix) - pathLen := len(p) - - if prefixLen > pathLen { - // Can't be a prefix if longer. - return false - } else if prefixLen == pathLen { - // Can be a prefix if they're equal, but otherwise no. - return p == prefix - } - - // otherPath is definitely shorter than p. - // We need to confirm that p[len(otherPath)] is a system separator. - - return strings.HasPrefix(p.ToString(), prefix.ToString()) && os.IsPathSeparator(p[prefixLen]) -} diff --git a/cli/internal/turbopath/absolute_system_path_darwin.go b/cli/internal/turbopath/absolute_system_path_darwin.go deleted file mode 100644 index e2c3bfffc89b1..0000000000000 --- a/cli/internal/turbopath/absolute_system_path_darwin.go +++ /dev/null @@ -1,23 +0,0 @@ -//go:build darwin -// +build darwin - -// Adapted from https://github.com/containerd/continuity/blob/b4ca35286886296377de39e6eafd1affae019fc3/driver/lchmod_unix.go -// Copyright The containerd Authors -// SPDX-License-Identifier: Apache-2.0 - -package turbopath - -import ( - "os" - - "golang.org/x/sys/unix" -) - -// Lchmod changes the mode of a file not following symlinks. -func (p AbsoluteSystemPath) Lchmod(mode os.FileMode) error { - err := unix.Fchmodat(unix.AT_FDCWD, p.ToString(), uint32(mode), unix.AT_SYMLINK_NOFOLLOW) - if err != nil { - err = &os.PathError{Op: "lchmod", Path: p.ToString(), Err: err} - } - return err -} diff --git a/cli/internal/turbopath/absolute_system_path_notdarwin.go b/cli/internal/turbopath/absolute_system_path_notdarwin.go deleted file mode 100644 index 11958886cad9b..0000000000000 --- a/cli/internal/turbopath/absolute_system_path_notdarwin.go +++ /dev/null @@ -1,13 +0,0 @@ -//go:build !darwin -// +build !darwin - -package turbopath - -import ( - "os" -) - -// Lchmod changes the mode of a file not following symlinks. -func (p AbsoluteSystemPath) Lchmod(mode os.FileMode) error { - return nil -} diff --git a/cli/internal/turbopath/anchored_system_path.go b/cli/internal/turbopath/anchored_system_path.go index 0957ead6f0a47..ff0296d3dd0e9 100644 --- a/cli/internal/turbopath/anchored_system_path.go +++ b/cli/internal/turbopath/anchored_system_path.go @@ -43,11 +43,6 @@ func (p AnchoredSystemPath) RestoreAnchor(anchor AbsoluteSystemPath) AbsoluteSys return AbsoluteSystemPath(filepath.Join(anchor.ToString(), p.ToString())) } -// Dir returns filepath.Dir for the path. -func (p AnchoredSystemPath) Dir() AnchoredSystemPath { - return AnchoredSystemPath(filepath.Dir(p.ToString())) -} - // Join appends relative path segments to this AnchoredSystemPath. func (p AnchoredSystemPath) Join(additional ...RelativeSystemPath) AnchoredSystemPath { cast := RelativeSystemPathArray(additional) diff --git a/cli/package.json b/cli/package.json index 6cca2b5464692..5ba7c26c04639 100644 --- a/cli/package.json +++ b/cli/package.json @@ -22,7 +22,6 @@ "globby": "11.1.0", "ngraph.generators": "^19.3.0", "shelljs": "^0.8.4", - "tar": "6.1.11", "uvu": "^0.5.3" } } diff --git a/cli/scripts/e2e/e2e.ts b/cli/scripts/e2e/e2e.ts index dc08e318f88f0..2d30434b67a6c 100644 --- a/cli/scripts/e2e/e2e.ts +++ b/cli/scripts/e2e/e2e.ts @@ -1,5 +1,4 @@ import execa from "execa"; -import tar from "tar"; import * as uvu from "uvu"; import * as assert from "uvu/assert"; import { Monorepo } from "../monorepo"; @@ -183,9 +182,8 @@ function runSmokeTests( const commandOutput = getCommandOutputAsArray(results); const hash = getHashFromOutput(commandOutput, "c#test"); assert.ok(!!hash, "No hash for c#test"); - const cacheItemPath = getCacheItemForHash(repo, hash); - await tar.x({ file: path.join(repo.root, cacheItemPath), cwd: repo.root }) const cachedLogFilePath = getCachedLogFilePathForTask( + getCachedDirForHash(repo, hash), path.join("packages", "c"), "test" ); @@ -215,9 +213,8 @@ function runSmokeTests( const commandOutput = getCommandOutputAsArray(results); const hash = getHashFromOutput(commandOutput, "c#lint"); assert.ok(!!hash, "No hash for c#lint"); - const cacheItemPath = getCacheItemForHash(repo, hash); - await tar.x({ file: path.join(repo.root, cacheItemPath), cwd: repo.root }) const cachedLogFilePath = getCachedLogFilePathForTask( + getCachedDirForHash(repo, hash), path.join("packages", "c"), "lint" ); @@ -695,19 +692,20 @@ function getHashFromOutput(lines: string[], taskId: string): string { return hash; } -function getCacheItemForHash(repo: Monorepo, hash: string): string { +function getCachedDirForHash(repo: Monorepo, hash: string): string { return path.join( repo.subdir ? repo.subdir : ".", "node_modules", ".cache", "turbo", - `${hash}.tar.gz` + hash ); } function getCachedLogFilePathForTask( + cacheDir: string, pathToPackage: string, taskName: string ): string { - return path.join(pathToPackage, ".turbo", `turbo-${taskName}.log`); + return path.join(cacheDir, pathToPackage, ".turbo", `turbo-${taskName}.log`); } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 7678f20cf8cce..a1c35b786202c 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -40,7 +40,6 @@ importers: globby: 11.1.0 ngraph.generators: ^19.3.0 shelljs: ^0.8.4 - tar: 6.1.11 uvu: ^0.5.3 devDependencies: copy-template-dir: 1.4.0 @@ -52,7 +51,6 @@ importers: globby: 11.1.0 ngraph.generators: 19.3.1 shelljs: 0.8.5 - tar: 6.1.11 uvu: 0.5.6 docs: @@ -2614,11 +2612,6 @@ packages: fsevents: 2.3.2 dev: true - /chownr/2.0.0: - resolution: {integrity: sha512-bIomtDF5KGpdogkLd9VspvFzk9KfpyyGlS8YFVZl7TGPBHL5snIOnxeshwVgPteQ9b4Eydl+pVbIyE1DcvCWgQ==} - engines: {node: '>=10'} - dev: true - /ci-info/3.4.0: resolution: {integrity: sha512-t5QdPT5jq3o262DOQ8zA6E1tlH2upmUc4Hlvrbx1pGYJuiiHl7O7rvVNI+l8HTVhd/q3Qc9vqimkNk5yiXsAug==} dev: true @@ -4387,13 +4380,6 @@ packages: universalify: 2.0.0 dev: true - /fs-minipass/2.1.0: - resolution: {integrity: sha512-V/JgOLFCS+R6Vcq0slCuaeWEdNC3ouDlJMNIsacH2VtALiu9mV4LPrHc5cDl8k5aw6J8jwgWWpiTo5RYhmIzvg==} - engines: {node: '>= 8'} - dependencies: - minipass: 3.3.4 - dev: true - /fs.realpath/1.0.0: resolution: {integrity: sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw==} @@ -6724,21 +6710,6 @@ packages: /minimist/1.2.6: resolution: {integrity: sha512-Jsjnk4bw3YJqYzbdyBiNsPWHPfO++UGG749Cxs6peCu5Xg4nrena6OVxOYxrQTqww0Jmwt+Ref8rggumkTLz9Q==} - /minipass/3.3.4: - resolution: {integrity: sha512-I9WPbWHCGu8W+6k1ZiGpPu0GkoKBeorkfKNuAFBNS1HNFJvke82sxvI5bzcCNpWPorkOO5QQ+zomzzwRxejXiw==} - engines: {node: '>=8'} - dependencies: - yallist: 4.0.0 - dev: true - - /minizlib/2.1.2: - resolution: {integrity: sha512-bAxsR8BVfj60DWXHE3u30oHzfl4G7khkSuPW+qvpd7jFRHm7dLxOjUk1EHACJ/hxLY8phGJ0YhYHZo7jil7Qdg==} - engines: {node: '>= 8'} - dependencies: - minipass: 3.3.4 - yallist: 4.0.0 - dev: true - /mixin-deep/1.3.2: resolution: {integrity: sha512-WRoDn//mXBiJ1H40rqa3vH0toePwSsGb45iInWlTySa+Uu4k3tYUSxa2v1KqAiLtvlrSzaExqS1gtk96A9zvEA==} engines: {node: '>=0.10.0'} @@ -6753,12 +6724,6 @@ packages: dependencies: minimist: 1.2.6 - /mkdirp/1.0.4: - resolution: {integrity: sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw==} - engines: {node: '>=10'} - hasBin: true - dev: true - /mri/1.2.0: resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} engines: {node: '>=4'} @@ -8536,18 +8501,6 @@ packages: - ts-node dev: true - /tar/6.1.11: - resolution: {integrity: sha512-an/KZQzQUkZCkuoAA64hM92X0Urb6VpRhAFllDzz44U2mcD5scmT3zBc4VgVpkugF580+DQn8eAFSyoQt0tznA==} - engines: {node: '>= 10'} - dependencies: - chownr: 2.0.0 - fs-minipass: 2.1.0 - minipass: 3.3.4 - minizlib: 2.1.2 - mkdirp: 1.0.4 - yallist: 4.0.0 - dev: true - /terminal-link/2.1.1: resolution: {integrity: sha512-un0FmiRUQNr5PJqy9kP7c40F5BOfpGlYTrxonDChEZB7pzZxRNp/bt+ymiy9/npwXya9KH99nJ/GXFIiUkYGFQ==} engines: {node: '>=8'}