Skip to content

Commit

Permalink
Use cacheitem to restore HTTP cache.
Browse files Browse the repository at this point in the history
  • Loading branch information
nathanhammond committed Apr 19, 2023
1 parent a24301d commit a1e5410
Show file tree
Hide file tree
Showing 4 changed files with 25 additions and 100 deletions.
98 changes: 3 additions & 95 deletions cli/internal/cache/cache_http.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"github.com/DataDog/zstd"

"github.com/vercel/turbo/cli/internal/analytics"
"github.com/vercel/turbo/cli/internal/cacheitem"
"github.com/vercel/turbo/cli/internal/tarpatch"
"github.com/vercel/turbo/cli/internal/turbopath"
)
Expand Down Expand Up @@ -251,102 +252,9 @@ func (cache *httpCache) retrieve(hash string) (bool, []turbopath.AnchoredSystemP
return true, files, duration, nil
}

// restoreTar returns posix-style repo-relative paths of the files it
// restored. In the future, these should likely be repo-relative system paths
// so that they are suitable for being fed into cache.Put for other caches.
// For now, I think this is working because windows also accepts /-delimited paths.
func restoreTar(root turbopath.AbsoluteSystemPath, reader io.Reader) ([]turbopath.AnchoredSystemPath, error) {
files := []turbopath.AnchoredSystemPath{}
missingLinks := []*tar.Header{}
zr := zstd.NewReader(reader)
var closeError error
defer func() { closeError = zr.Close() }()
tr := tar.NewReader(zr)
for {
hdr, err := tr.Next()
if err != nil {
if err == io.EOF {
for _, link := range missingLinks {
err := restoreSymlink(root, link, true)
if err != nil {
return nil, err
}
}

return files, closeError
}
return nil, err
}
// hdr.Name is always a posix-style path
// FIXME: THIS IS A BUG.
restoredName := turbopath.AnchoredUnixPath(hdr.Name)
files = append(files, restoredName.ToSystemPath())
filename := restoredName.ToSystemPath().RestoreAnchor(root)
if isChild, err := root.ContainsPath(filename); err != nil {
return nil, err
} else if !isChild {
return nil, fmt.Errorf("cannot untar file to %v", filename)
}
switch hdr.Typeflag {
case tar.TypeDir:
if err := filename.MkdirAll(0775); err != nil {
return nil, err
}
case tar.TypeReg:
if dir := filename.Dir(); dir != "." {
if err := dir.MkdirAll(0775); err != nil {
return nil, err
}
}
if f, err := filename.OpenFile(os.O_WRONLY|os.O_TRUNC|os.O_CREATE, os.FileMode(hdr.Mode)); err != nil {
return nil, err
} else if _, err := io.Copy(f, tr); err != nil {
return nil, err
} else if err := f.Close(); err != nil {
return nil, err
}
case tar.TypeSymlink:
if err := restoreSymlink(root, hdr, false); errors.Is(err, errNonexistentLinkTarget) {
missingLinks = append(missingLinks, hdr)
} else if err != nil {
return nil, err
}
default:
log.Printf("Unhandled file type %d for %s", hdr.Typeflag, hdr.Name)
}
}
}

var errNonexistentLinkTarget = errors.New("the link target does not exist")

func restoreSymlink(root turbopath.AbsoluteSystemPath, hdr *tar.Header, allowNonexistentTargets bool) error {
// Note that hdr.Linkname is really the link target
relativeLinkTarget := filepath.FromSlash(hdr.Linkname)
linkFilename := root.UntypedJoin(hdr.Name)
if err := linkFilename.EnsureDir(); err != nil {
return err
}

// TODO: check if this is an absolute path, or if we even care
linkTarget := linkFilename.Dir().UntypedJoin(relativeLinkTarget)
if _, err := linkTarget.Lstat(); err != nil {
if os.IsNotExist(err) {
if !allowNonexistentTargets {
return errNonexistentLinkTarget
}
// if we're allowing nonexistent link targets, proceed to creating the link
} else {
return err
}
}
// Ensure that the link we're about to create doesn't already exist
if err := linkFilename.Remove(); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
if err := linkFilename.Symlink(relativeLinkTarget); err != nil {
return err
}
return nil
cache := cacheitem.FromReader(reader, true)
return cache.Restore(root)
}

func (cache *httpCache) Clean(_ turbopath.AbsoluteSystemPath) {
Expand Down
12 changes: 8 additions & 4 deletions cli/internal/cacheitem/cacheitem.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@ import (
"crypto/sha512"
"errors"
"io"
"os"

"github.com/vercel/turbo/cli/internal/turbopath"
)
Expand All @@ -32,7 +31,7 @@ type CacheItem struct {
tw *tar.Writer
zw io.WriteCloser
fileBuffer *bufio.Writer
handle *os.File
handle io.Reader
compressed bool
}

Expand All @@ -57,9 +56,14 @@ func (ci *CacheItem) Close() error {
}

if ci.handle != nil {
if err := ci.handle.Close(); err != nil {
return err
closer, isCloser := ci.handle.(io.Closer)

if isCloser {
if err := closer.Close(); err != nil {
return err
}
}

}

return nil
Expand Down
7 changes: 6 additions & 1 deletion cli/internal/cacheitem/create.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,12 @@ func Create(path turbopath.AbsoluteSystemPath) (*CacheItem, error) {
// Wires all the writers end-to-end:
// tar.Writer -> zstd.Writer -> fileBuffer -> file
func (ci *CacheItem) init() {
fileBuffer := bufio.NewWriterSize(ci.handle, 2^20) // Flush to disk in 1mb chunks.
writer, isWriter := ci.handle.(io.Writer)
if !isWriter {
panic("can't write to this cache item")
}

fileBuffer := bufio.NewWriterSize(writer, 2^20) // Flush to disk in 1mb chunks.

var tw *tar.Writer
if ci.compressed {
Expand Down
8 changes: 8 additions & 0 deletions cli/internal/cacheitem/restore.go
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,14 @@ import (
"github.com/vercel/turbo/cli/internal/turbopath"
)

// FromReader returns an existing CacheItem at the specified path.
func FromReader(reader io.Reader, compressed bool) *CacheItem {
return &CacheItem{
handle: reader,
compressed: compressed,
}
}

// Open returns an existing CacheItem at the specified path.
func Open(path turbopath.AbsoluteSystemPath) (*CacheItem, error) {
handle, err := sequential.OpenFile(path.ToString(), os.O_RDONLY, 0777)
Expand Down

0 comments on commit a1e5410

Please sign in to comment.