Revert "Caching, but to tar files."

cli/go.mod

@@ -25,7 +25,6 @@ require (
 	github.com/mitchellh/cli v1.1.2
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/mitchellh/mapstructure v1.5.0
-	github.com/moby/sys/sequential v0.5.0
 	github.com/nightlyone/lockfile v1.0.0
 	github.com/pkg/errors v0.9.1
 	github.com/pyr-sh/dag v1.0.0
@@ -37,7 +36,6 @@ require (
 	github.com/stretchr/testify v1.8.0
 	github.com/yookoala/realpath v1.0.0
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
-	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a
 	google.golang.org/grpc v1.46.2
 	google.golang.org/protobuf v1.28.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -81,6 +79,7 @@ require (
 	github.com/subosito/gotenv v1.3.0 // indirect
 	golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4 // indirect
 	golang.org/x/net v0.0.0-20220520000938-2e3eb7b945c2 // indirect
+	golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a // indirect
 	golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 // indirect
 	golang.org/x/text v0.3.7 // indirect
 	google.golang.org/genproto v0.0.0-20220519153652-3a47de7e79bd // indirect

cli/go.sum

@@ -360,8 +360,6 @@ github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RR
 github.com/mitchellh/reflectwalk v1.0.0/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
 github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY70sEEpE=
 github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
-github.com/moby/sys/sequential v0.5.0 h1:OPvI35Lzn9K04PBbCLW0g4LcFAJgHsvXsRyewg5lXtc=
-github.com/moby/sys/sequential v0.5.0/go.mod h1:tH2cOOs5V9MlPiXcQzRC+eEyab644PWKGRYaaV5ZZlo=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
 github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=

cli/internal/cache/cache_fs.go

@@ -1,29 +1,30 @@
 // Adapted from https://github.com/thought-machine/please
 // Copyright Thought Machine, Inc. or its affiliates. All Rights Reserved.
 // SPDX-License-Identifier: Apache-2.0
-
-// Package cache implements our cache abstraction.
 package cache
 
 import (
 	"encoding/json"
 	"fmt"
+	"runtime"
 
 	"github.com/vercel/turborepo/cli/internal/analytics"
-	"github.com/vercel/turborepo/cli/internal/cacheitem"
+	"github.com/vercel/turborepo/cli/internal/fs"
 	"github.com/vercel/turborepo/cli/internal/turbopath"
+	"golang.org/x/sync/errgroup"
 )
 
 // fsCache is a local filesystem cache
 type fsCache struct {
 	cacheDirectory turbopath.AbsoluteSystemPath
 	recorder       analytics.Recorder
+	repoRoot       turbopath.AbsoluteSystemPath
 }
 
 // newFsCache creates a new filesystem cache
 func newFsCache(opts Opts, recorder analytics.Recorder, repoRoot turbopath.AbsoluteSystemPath) (*fsCache, error) {
 	cacheDir := opts.ResolveCacheDir(repoRoot)
-	if err := cacheDir.MkdirAll(0775); err != nil {
+	if err := cacheDir.MkdirAll(); err != nil {
 		return nil, err
 	}
 	return &fsCache{
@@ -34,44 +35,33 @@ func newFsCache(opts Opts, recorder analytics.Recorder, repoRoot turbopath.Absol
 
 // Fetch returns true if items are cached. It moves them into position as a side effect.
 func (f *fsCache) Fetch(anchor turbopath.AbsoluteSystemPath, hash string, _unusedOutputGlobs []string) (bool, []turbopath.AnchoredSystemPath, int, error) {
-	cachePath := f.cacheDirectory.UntypedJoin(hash + ".tar.gz")
+	cachedFolder := f.cacheDirectory.UntypedJoin(hash)
 
 	// If it's not in the cache bail now
-	if !cachePath.FileExists() {
+	if !cachedFolder.DirExists() {
 		f.logFetch(false, hash, 0)
 		return false, nil, 0, nil
 	}
 
-	cacheItem, openErr := cacheitem.Open(cachePath)
-	if openErr != nil {
-		return false, nil, 0, openErr
-	}
-
-	restoredFiles, restoreErr := cacheItem.Restore(anchor)
-	if restoreErr != nil {
-		_ = cacheItem.Close()
-		return false, nil, 0, restoreErr
+	// Otherwise, copy it into position
+	err := fs.RecursiveCopy(cachedFolder.ToStringDuringMigration(), anchor.ToStringDuringMigration())
+	if err != nil {
+		// TODO: what event to log here?
+		return false, nil, 0, fmt.Errorf("error moving artifact from cache into %v: %w", anchor, err)
 	}
 
 	meta, err := ReadCacheMetaFile(f.cacheDirectory.UntypedJoin(hash + "-meta.json"))
 	if err != nil {
-		_ = cacheItem.Close()
 		return false, nil, 0, fmt.Errorf("error reading cache metadata: %w", err)
 	}
 	f.logFetch(true, hash, meta.Duration)
-
-	// Wait to see what happens with close.
-	closeErr := cacheItem.Close()
-	if closeErr != nil {
-		return false, restoredFiles, 0, closeErr
-	}
-	return true, restoredFiles, meta.Duration, nil
+	return true, nil, meta.Duration, nil
 }
 
 func (f *fsCache) Exists(hash string) (ItemStatus, error) {
-	cachePath := f.cacheDirectory.UntypedJoin(hash + "tar.gz")
+	cachedFolder := f.cacheDirectory.UntypedJoin(hash)
 
-	if !cachePath.FileExists() {
+	if !cachedFolder.DirExists() {
 		return ItemStatus{Local: false}, nil
 	}
 
@@ -95,18 +85,40 @@ func (f *fsCache) logFetch(hit bool, hash string, duration int) {
 }
 
 func (f *fsCache) Put(anchor turbopath.AbsoluteSystemPath, hash string, duration int, files []turbopath.AnchoredSystemPath) error {
-	cachePath := f.cacheDirectory.UntypedJoin(hash + ".tar.gz")
-	cacheItem, err := cacheitem.Create(cachePath)
-	if err != nil {
-		return err
+	g := new(errgroup.Group)
+
+	numDigesters := runtime.NumCPU()
+	fileQueue := make(chan turbopath.AnchoredSystemPath, numDigesters)
+
+	for i := 0; i < numDigesters; i++ {
+		g.Go(func() error {
+			for file := range fileQueue {
+				statedFile := fs.LstatCachedFile{Path: file.RestoreAnchor(anchor)}
+				fromType, err := statedFile.GetType()
+				if err != nil {
+					return fmt.Errorf("error stat'ing cache source %v: %v", file, err)
+				}
+				if !fromType.IsDir() {
+					if err := f.cacheDirectory.UntypedJoin(hash, file.ToStringDuringMigration()).EnsureDir(); err != nil {
+						return fmt.Errorf("error ensuring directory file from cache: %w", err)
+					}
+
+					if err := fs.CopyFile(&statedFile, f.cacheDirectory.UntypedJoin(hash, file.ToStringDuringMigration()).ToStringDuringMigration()); err != nil {
+						return fmt.Errorf("error copying file from cache: %w", err)
+					}
+				}
+			}
+			return nil
+		})
 	}
 
 	for _, file := range files {
-		err := cacheItem.AddFile(anchor, file)
-		if err != nil {
-			_ = cacheItem.Close()
-			return err
-		}
+		fileQueue <- file
+	}
+	close(fileQueue)
+
+	if err := g.Wait(); err != nil {
+		return err
 	}
 
 	writeErr := WriteCacheMetaFile(f.cacheDirectory.UntypedJoin(hash+"-meta.json"), &CacheMetadata{
@@ -115,11 +127,10 @@ func (f *fsCache) Put(anchor turbopath.AbsoluteSystemPath, hash string, duration
 	})
 
 	if writeErr != nil {
-		_ = cacheItem.Close()
 		return writeErr
 	}
 
-	return cacheItem.Close()
+	return nil
 }
 
 func (f *fsCache) Clean(anchor turbopath.AbsoluteSystemPath) {
@@ -130,7 +141,7 @@ func (f *fsCache) CleanAll() {
 	fmt.Println("Not implemented yet")
 }
 
-func (f *fsCache) Shutdown() {}
+func (cache *fsCache) Shutdown() {}
 
 // CacheMetadata stores duration and hash information for a cache entry so that aggregate Time Saved calculations
 // can be made from artifacts from various caches

cli/internal/cache/cache_fs_test.go

@@ -1,11 +1,12 @@
 package cache
 
 import (
+	"io/ioutil"
+	"os"
 	"path/filepath"
 	"testing"
 
 	"github.com/vercel/turborepo/cli/internal/analytics"
-	"github.com/vercel/turborepo/cli/internal/cacheitem"
 	"github.com/vercel/turborepo/cli/internal/turbopath"
 	"gotest.tools/v3/assert"
 )
@@ -32,7 +33,7 @@ func TestPut(t *testing.T) {
 
 	src := turbopath.AbsoluteSystemPath(t.TempDir())
 	childDir := src.UntypedJoin("child")
-	err := childDir.MkdirAll(0775)
+	err := childDir.MkdirAll()
 	assert.NilError(t, err, "Mkdir")
 	aPath := childDir.UntypedJoin("a")
 	aFile, err := aPath.Create()
@@ -58,6 +59,7 @@ func TestPut(t *testing.T) {
 	assert.NilError(t, circlePath.Symlink(filepath.FromSlash("../child")), "Symlink")
 
 	files := []turbopath.AnchoredSystemPath{
+		turbopath.AnchoredUnixPath(".").ToSystemPath(),            // src
 		turbopath.AnchoredUnixPath("child/").ToSystemPath(),       // childDir
 		turbopath.AnchoredUnixPath("child/a").ToSystemPath(),      // aPath,
 		turbopath.AnchoredUnixPath("b").ToSystemPath(),            // bPath,
@@ -76,26 +78,17 @@ func TestPut(t *testing.T) {
 
 	hash := "the-hash"
 	duration := 0
-	putErr := cache.Put(src, hash, duration, files)
-	assert.NilError(t, putErr, "Put")
+	err = cache.Put(src, hash, duration, files)
+	assert.NilError(t, err, "Put")
 
 	// Verify that we got the files that we're expecting
 	dstCachePath := dst.UntypedJoin(hash)
 
-	// This test checks outputs, so we go ahead and pull things back out.
-	// Attempting to satisfy our beliefs that the change is viable with
-	// as few changes to the tests as possible.
-	cacheItem, openErr := cacheitem.Open(dst.UntypedJoin(hash + ".tar.gz"))
-	assert.NilError(t, openErr, "Open")
-
-	_, restoreErr := cacheItem.Restore(dstCachePath)
-	assert.NilError(t, restoreErr, "Restore")
-
 	dstAPath := dstCachePath.UntypedJoin("child", "a")
-	assertFileMatches(t, aPath, dstAPath)
+	assertFileMatches(t, aPath.ToStringDuringMigration(), dstAPath.ToStringDuringMigration())
 
 	dstBPath := dstCachePath.UntypedJoin("b")
-	assertFileMatches(t, bPath, dstBPath)
+	assertFileMatches(t, bPath.ToStringDuringMigration(), dstBPath.ToStringDuringMigration())
 
 	dstLinkPath := dstCachePath.UntypedJoin("child", "link")
 	target, err := dstLinkPath.Readlink()
@@ -118,20 +111,18 @@ func TestPut(t *testing.T) {
 	if circleLinkDest != expectedCircleLinkDest {
 		t.Errorf("Cache link got %v, want %v", circleLinkDest, expectedCircleLinkDest)
 	}
-
-	assert.NilError(t, cacheItem.Close(), "Close")
 }
 
-func assertFileMatches(t *testing.T, orig turbopath.AbsoluteSystemPath, copy turbopath.AbsoluteSystemPath) {
+func assertFileMatches(t *testing.T, orig string, copy string) {
 	t.Helper()
-	origBytes, err := orig.ReadFile()
+	origBytes, err := ioutil.ReadFile(orig)
 	assert.NilError(t, err, "ReadFile")
-	copyBytes, err := copy.ReadFile()
+	copyBytes, err := ioutil.ReadFile(copy)
 	assert.NilError(t, err, "ReadFile")
 	assert.DeepEqual(t, origBytes, copyBytes)
-	origStat, err := orig.Lstat()
+	origStat, err := os.Lstat(orig)
 	assert.NilError(t, err, "Lstat")
-	copyStat, err := copy.Lstat()
+	copyStat, err := os.Lstat(copy)
 	assert.NilError(t, err, "Lstat")
 	assert.Equal(t, origStat.Mode(), copyStat.Mode())
 }
@@ -157,13 +148,12 @@ func TestFetch(t *testing.T) {
 	// "some-package"/...
 
 	cacheDir := turbopath.AbsoluteSystemPath(t.TempDir())
-	hash := "the-hash"
-	src := cacheDir.UntypedJoin(hash, "some-package")
-	err := src.MkdirAll(0775)
+	src := cacheDir.UntypedJoin("the-hash", "some-package")
+	err := src.MkdirAll()
 	assert.NilError(t, err, "mkdirAll")
 
 	childDir := src.UntypedJoin("child")
-	err = childDir.MkdirAll(0775)
+	err = childDir.MkdirAll()
 	assert.NilError(t, err, "Mkdir")
 	aPath := childDir.UntypedJoin("a")
 	aFile, err := aPath.Create()
@@ -184,12 +174,9 @@ func TestFetch(t *testing.T) {
 	assert.NilError(t, srcLinkPath.Symlink(linkTarget), "Symlink")
 
 	srcBrokenLinkPath := childDir.UntypedJoin("broken")
-	srcBrokenLinkTarget := turbopath.AnchoredUnixPath("missing").ToSystemPath()
-	assert.NilError(t, srcBrokenLinkPath.Symlink(srcBrokenLinkTarget.ToString()), "Symlink")
-
+	assert.NilError(t, srcBrokenLinkPath.Symlink("missing"), "Symlink")
 	circlePath := childDir.Join("circle")
-	srcCircleLinkTarget := turbopath.AnchoredUnixPath("../child").ToSystemPath()
-	assert.NilError(t, circlePath.Symlink(srcCircleLinkTarget.ToString()), "Symlink")
+	assert.NilError(t, circlePath.Symlink(filepath.FromSlash("../child")), "Symlink")
 
 	metadataPath := cacheDir.UntypedJoin("the-hash-meta.json")
 	err = metadataPath.WriteFile([]byte(`{"hash":"the-hash","duration":0}`), 0777)
@@ -202,51 +189,46 @@ func TestFetch(t *testing.T) {
 		recorder:       dr,
 	}
 
-	inputFiles := []turbopath.AnchoredSystemPath{
-		turbopath.AnchoredUnixPath("some-package/child/").ToSystemPath(),       // childDir
-		turbopath.AnchoredUnixPath("some-package/child/a").ToSystemPath(),      // aPath,
-		turbopath.AnchoredUnixPath("some-package/b").ToSystemPath(),            // bPath,
-		turbopath.AnchoredUnixPath("some-package/child/link").ToSystemPath(),   // srcLinkPath,
-		turbopath.AnchoredUnixPath("some-package/child/broken").ToSystemPath(), // srcBrokenLinkPath,
-		turbopath.AnchoredUnixPath("some-package/child/circle").ToSystemPath(), // circlePath
-	}
-
-	putErr := cache.Put(cacheDir.UntypedJoin(hash), hash, 0, inputFiles)
-	assert.NilError(t, putErr, "Put")
-
 	outputDir := turbopath.AbsoluteSystemPath(t.TempDir())
 	dstOutputPath := "some-package"
 	hit, files, _, err := cache.Fetch(outputDir, "the-hash", []string{})
 	assert.NilError(t, err, "Fetch")
 	if !hit {
 		t.Error("Fetch got false, want true")
 	}
-	if len(files) != len(inputFiles) {
-		t.Errorf("len(files) got %v, want %v", len(files), len(inputFiles))
+	if len(files) != 0 {
+		// Not for any particular reason, but currently the fs cache doesn't return the
+		// list of files copied
+		t.Errorf("len(files) got %v, want 0", len(files))
 	}
+	t.Logf("files %v", files)
 
-	dstAPath := outputDir.UntypedJoin(dstOutputPath, "child", "a")
-	assertFileMatches(t, aPath, dstAPath)
+	dstAPath := filepath.Join(outputDir.ToStringDuringMigration(), dstOutputPath, "child", "a")
+	assertFileMatches(t, aPath.ToStringDuringMigration(), dstAPath)
 
-	dstBPath := outputDir.UntypedJoin(dstOutputPath, "b")
-	assertFileMatches(t, bPath, dstBPath)
+	dstBPath := filepath.Join(outputDir.ToStringDuringMigration(), dstOutputPath, "b")
+	assertFileMatches(t, bPath.ToStringDuringMigration(), dstBPath)
 
-	dstLinkPath := outputDir.UntypedJoin(dstOutputPath, "child", "link")
-	target, err := dstLinkPath.Readlink()
+	dstLinkPath := filepath.Join(outputDir.ToStringDuringMigration(), dstOutputPath, "child", "link")
+	target, err := os.Readlink(dstLinkPath)
 	assert.NilError(t, err, "Readlink")
 	if target != linkTarget {
 		t.Errorf("Readlink got %v, want %v", target, linkTarget)
 	}
 
-	// Assert that we restore broken symlinks correctly
-	dstBrokenLinkPath := outputDir.UntypedJoin(dstOutputPath, "child", "broken")
-	target, readlinkErr := dstBrokenLinkPath.Readlink()
-	assert.NilError(t, readlinkErr, "Readlink")
-	assert.Equal(t, target, srcBrokenLinkTarget.ToString())
-
-	// Assert that we restore symlinks to directories correctly
-	dstCirclePath := outputDir.UntypedJoin(dstOutputPath, "child", "circle")
-	circleTarget, circleReadlinkErr := dstCirclePath.Readlink()
-	assert.NilError(t, circleReadlinkErr, "Circle Readlink")
-	assert.Equal(t, circleTarget, srcCircleLinkTarget.ToString())
+	// We currently don't restore broken symlinks. This is probably a bug
+	dstBrokenLinkPath := filepath.Join(outputDir.ToStringDuringMigration(), dstOutputPath, "child", "broken")
+	_, err = os.Readlink(dstBrokenLinkPath)
+	assert.ErrorIs(t, err, os.ErrNotExist)
+
+	// Currently, on restore, we convert symlink-to-directory to empty-directory
+	// This is very likely not ideal behavior, but leaving this test here to verify
+	// that it is what we expect at this point in time.
+	dstCirclePath := filepath.Join(outputDir.ToStringDuringMigration(), dstOutputPath, "child", "circle")
+	circleStat, err := os.Lstat(dstCirclePath)
+	assert.NilError(t, err, "Lstat")
+	assert.Equal(t, circleStat.IsDir(), true)
+	entries, err := os.ReadDir(dstCirclePath)
+	assert.NilError(t, err, "ReadDir")
+	assert.Equal(t, len(entries), 0)
 }