Skip to content

Commit

Permalink
Merge pull request #418 from vmware-tanzu/imgpkg-resume-tar-flag
Browse files Browse the repository at this point in the history 
Add feature to allow resume of copy to tar
  • Loading branch information
@joaopapereira
joaopapereira committed Jul 28, 2022
2 parents af1b1ab + 5ab79e0 commit c6fae29
Show file tree
Hide file tree
Showing 11 changed files with 375 additions and 39 deletions.
6 changes: 5 additions & 1 deletion pkg/imgpkg/cmd/copy.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -146,9 +146,13 @@ func (c *CopyOptions) Run() error {
if c.LockOutputFlags.LockFilePath != "" {
return fmt.Errorf("Cannot output lock file with tar destination")
}
return repoSrc.CopyToTar(c.TarFlags.TarDst)
return repoSrc.CopyToTar(c.TarFlags.TarDst, c.TarFlags.Resume)

case c.isRepoDst():
if c.TarFlags.Resume {
return fmt.Errorf("Flag --resume can only be used when copying to tar")
}

processedImages, err := repoSrc.CopyToRepo(c.RepoDst)
if err != nil {
return err
Expand Down
7 changes: 4 additions & 3 deletions pkg/imgpkg/cmd/copy_repo_src.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,16 +36,17 @@ type CopyRepoSrc struct {
signatureRetriever SignatureRetriever
}

func (c CopyRepoSrc) CopyToTar(dstPath string) error {
// CopyToTar copies image or bundle into the provided path
func (c CopyRepoSrc) CopyToTar(dstPath string, resume bool) error {
c.ui.Tracef("CopyToTar\n")

unprocessedImageRefs, _, err := c.getAllSourceImages()
if err != nil {
return err
}

ids, err := c.tarImageSet.Export(unprocessedImageRefs, dstPath, c.registry,
imagetar.NewImageLayerWriterCheck(c.IncludeNonDistributable))
c.ui.Tracef("Exporting images to tar\n")
ids, err := c.tarImageSet.Export(unprocessedImageRefs, dstPath, c.registry, imagetar.NewImageLayerWriterCheck(c.IncludeNonDistributable), resume)
if err != nil {
return err
}
Expand Down
109 changes: 93 additions & 16 deletions pkg/imgpkg/cmd/copy_repo_src_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import (
"net/http"
"os"
"path/filepath"
"regexp"
"strings"
"testing"

Expand Down Expand Up @@ -79,7 +80,7 @@ func TestToTarBundle(t *testing.T) {
bundleTarPath := filepath.Join(os.TempDir(), "bundle.tar")
defer os.Remove(bundleTarPath)

err := subject.CopyToTar(bundleTarPath)
err := subject.CopyToTar(bundleTarPath, false)
require.NoError(t, err)

assertTarballContainsEveryLayer(t, bundleTarPath)
Expand Down Expand Up @@ -117,7 +118,7 @@ bundle:
bundleTarPath := filepath.Join(os.TempDir(), "bundle.tar")
defer os.Remove(bundleTarPath)

err = subject.CopyToTar(bundleTarPath)
err = subject.CopyToTar(bundleTarPath, false)
require.NoError(t, err)

assertTarballContainsOnlyDistributableLayers(bundleTarPath, t)
Expand Down Expand Up @@ -148,7 +149,7 @@ func TestToTarBundleContainingNonDistributableLayers(t *testing.T) {
imageTarPath := filepath.Join(os.TempDir(), "bundle.tar")
defer os.Remove(imageTarPath)

err := subject.CopyToTar(imageTarPath)
err := subject.CopyToTar(imageTarPath, false)
require.NoError(t, err)

assertTarballContainsOnlyDistributableLayers(imageTarPath, t)
Expand All @@ -160,7 +161,7 @@ func TestToTarBundleContainingNonDistributableLayers(t *testing.T) {
imageTarPath := filepath.Join(os.TempDir(), "bundle.tar")
defer os.Remove(imageTarPath)

err := subject.CopyToTar(imageTarPath)
err := subject.CopyToTar(imageTarPath, false)
require.NoError(t, err)

digest, err := nonDistributableLayer.Digest()
Expand All @@ -179,7 +180,7 @@ func TestToTarBundleContainingNonDistributableLayers(t *testing.T) {
imageTarPath := filepath.Join(os.TempDir(), "bundle.tar")
defer os.Remove(imageTarPath)

err := subject.CopyToTar(imageTarPath)
err := subject.CopyToTar(imageTarPath, false)
require.NoError(t, err)

assertTarballContainsEveryLayer(t, imageTarPath)
Expand All @@ -193,7 +194,7 @@ func TestToTarBundleContainingNonDistributableLayers(t *testing.T) {
imageTarPath := filepath.Join(os.TempDir(), "bundle.tar")
defer os.Remove(imageTarPath)

err := subject.CopyToTar(imageTarPath)
err := subject.CopyToTar(imageTarPath, false)
require.NoError(t, err)

assert.NotContains(t, stdOut.String(), "Warning: '--include-non-distributable-layers' flag provided, but no images contained a non-distributable layer.")
Expand Down Expand Up @@ -223,7 +224,7 @@ images:
imageTarPath := filepath.Join(os.TempDir(), "bundle.tar")
defer os.Remove(imageTarPath)

err := subject.CopyToTar(imageTarPath)
err := subject.CopyToTar(imageTarPath, false)
require.NoError(t, err)

assertTarballContainsOnlyDistributableLayers(imageTarPath, t)
Expand All @@ -232,8 +233,10 @@ images:

func TestToTarImage(t *testing.T) {
imageName := "library/image"
randomImageName := "my/image/one"
fakeRegistry := helpers.NewFakeRegistry(t, &helpers.Logger{LogLevel: helpers.LogDebug})
fakeRegistry.WithImageFromPath(imageName, "test_assets/image_with_config", map[string]string{})
fakeRegistry.WithRandomImageWithLayers(randomImageName, 20)
defer fakeRegistry.CleanUp()

subject := subject
Expand All @@ -246,7 +249,7 @@ func TestToTarImage(t *testing.T) {
imageTarPath := filepath.Join(os.TempDir(), "bundle.tar")
defer os.Remove(imageTarPath)

err := subject.CopyToTar(imageTarPath)
err := subject.CopyToTar(imageTarPath, false)
require.NoError(t, err)

assertTarballContainsEveryLayer(t, imageTarPath)
Expand All @@ -259,7 +262,7 @@ func TestToTarImage(t *testing.T) {
imageTarPath := filepath.Join(os.TempDir(), "bundle.tar")
defer os.Remove(imageTarPath)

err := subject.CopyToTar(imageTarPath)
err := subject.CopyToTar(imageTarPath, false)
require.NoError(t, err)

assertTarballContainsEveryLayer(t, imageTarPath)
Expand All @@ -273,10 +276,84 @@ func TestToTarImage(t *testing.T) {
imageTarPath := filepath.Join(os.TempDir(), "bundle.tar")
defer os.Remove(imageTarPath)

err := subject.CopyToTar(imageTarPath)
err := subject.CopyToTar(imageTarPath, false)
require.NoError(t, err)

assert.Contains(t, stdOut.String(), "Warning: '--include-non-distributable-layers' flag provided, but no images contained a non-distributable layer.\n")
require.Contains(t, stdOut.String(), "Warning: '--include-non-distributable-layers' flag provided, but no images contained a non-distributable layer.\n")
})

t.Run("When copy to tar fails the first time but second call with resume completes successfully", func(t *testing.T) {
numberOfRequests := 0
var failedDigest regv1.Hash
var layersInTar []regv1.Layer
fakeRegistry.WithCustomHandler(func(writer http.ResponseWriter, request *http.Request) bool {
matched, err := regexp.MatchString("/v2/.+/blobs", request.URL.Path)
require.NoError(t, err)

if matched && request.Method == "GET" {
parts := strings.Split(request.URL.Path, "/")
sha := parts[len(parts)-1]
hash, err := regv1.NewHash(sha)
// This loop ensures that if a layer is in the tar already we can return gibberish\
// because imgpkg is not going to use this information
for _, layer := range layersInTar {
digest, err := layer.Digest()
require.NoError(t, err)
if hash.String() == digest.String() {
size, err := layer.Size()
require.NoError(t, err)
bs := make([]byte, size)
writer.Write(bs)
return true
}
}

numberOfRequests++
// This if statement makes sure that in the 5th layer that imgpkg tries to retrieve some gibberish
// is returned to simulate a failure in communication
if numberOfRequests == 5 {
require.NoError(t, err)
if !fakeRegistry.IsConfigBlobLayer(hash) {
layer, err := fakeRegistry.Layer(hash)
require.NoError(t, err)
size, err := layer.Size()
require.NoError(t, err)
writer.WriteHeader(http.StatusOK)
bs := make([]byte, size)
writer.Write(bs)
failedDigest = hash
return true
}

// Found configuration layer, it doesn't count as content blob
numberOfRequests--
}
}

return false
})

imageTarPath := filepath.Join(os.TempDir(), " imgpkg-test-img.tar")
if _, err := os.Stat(imageTarPath); err == nil {
os.Remove(imageTarPath)
}
defer os.Remove(imageTarPath)

subject := subject
subject.ImageFlags = ImageFlags{
fakeRegistry.ReferenceOnTestServer(randomImageName),
}
err := subject.CopyToTar(imageTarPath, false)
require.ErrorContains(t, err, "error verifying sha256 checksum")
layersInTar, err = imagetar.NewTarReader(imageTarPath).PresentLayers()
require.NoError(t, err)
require.Greater(t, len(layersInTar), 1)
require.NotContains(t, layersInTar, failedDigest, "tar should not contain the layer that fails to download")

err = subject.CopyToTar(imageTarPath, true)
require.NoError(t, err)

assertTarballContainsEveryLayer(t, imageTarPath)
})
}

Expand All @@ -296,7 +373,7 @@ func TestToTarImageContainingNonDistributableLayers(t *testing.T) {
imageTarPath := filepath.Join(os.TempDir(), "bundle.tar")
defer os.Remove(imageTarPath)

err := subject.CopyToTar(imageTarPath)
err := subject.CopyToTar(imageTarPath, false)
if err != nil {
t.Fatalf("Expected CopyToTar() to succeed but got: %s", err)
}
Expand All @@ -310,7 +387,7 @@ func TestToTarImageContainingNonDistributableLayers(t *testing.T) {
imageTarPath := filepath.Join(os.TempDir(), "bundle.tar")
defer os.Remove(imageTarPath)

err := subject.CopyToTar(imageTarPath)
err := subject.CopyToTar(imageTarPath, false)
if err != nil {
t.Fatalf("Expected CopyToTar() to succeed but got: %s", err)
}
Expand All @@ -335,7 +412,7 @@ func TestToTarImageIndex(t *testing.T) {
imageTarPath := filepath.Join(os.TempDir(), "bundle.tar")
defer os.Remove(imageTarPath)

err := subject.CopyToTar(imageTarPath)
err := subject.CopyToTar(imageTarPath, false)
assert.NoError(t, err)

assertTarballContainsEveryImageInImageIndex(t, imageTarPath, int(numOfImagesForImageIndex))
Expand Down Expand Up @@ -642,7 +719,7 @@ func TestToRepoBundleCreatesValidLocationOCI(t *testing.T) {
destRepo := fakeRegistry.ReferenceOnTestServer("library/bundle-copy")

logger.Section("create Tar file with bundle", func() {
err := subject.CopyToTar(tarFile)
err := subject.CopyToTar(tarFile, false)
require.NoError(t, err)
})

Expand Down Expand Up @@ -781,7 +858,7 @@ func TestToRepoFromTar(t *testing.T) {
subject.registry = fakeRegistry.Build()

logger.Section("create Tar file with bundle", func() {
err := subject.CopyToTar(tarFile)
err := subject.CopyToTar(tarFile, false)
require.NoError(t, err)
})

Expand Down
2 changes: 2 additions & 0 deletions pkg/imgpkg/cmd/tar_flags.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,11 +10,13 @@ import (
type TarFlags struct {
TarSrc string
TarDst string
Resume bool
}

func (t *TarFlags) Set(cmd *cobra.Command) {
cmd.Flags().StringVar(&t.TarDst, "to-tar", "", "Location to write a tar file containing assets")
cmd.Flags().StringVar(&t.TarSrc, "tar", "", "Path to tar file which contains assets to be copied to a registry")
cmd.Flags().BoolVar(&t.Resume, "resume", false, "Resume the copy to tar. When set to true will try to read the tar and only download the missing blobs")
}

func (t TarFlags) IsSrc() bool { return t.TarSrc != "" }
Expand Down
12 changes: 12 additions & 0 deletions pkg/imgpkg/imagedesc/described_image_index.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ func (i DescribedImageIndex) IndexManifest() (*regv1.IndexManifest, error) {
return manifest, nil
}

// Image retrieve the image that matches the provided digest
func (i DescribedImageIndex) Image(digest regv1.Hash) (regv1.Image, error) {
for _, img := range i.images {
imgDigest, err := img.Digest()
Expand All @@ -61,6 +62,17 @@ func (i DescribedImageIndex) Image(digest regv1.Hash) (regv1.Image, error) {
return nil, fmt.Errorf("Expected to find image '%s' by digest", digest)
}

// Images retrieve all images associated with the described image
func (i DescribedImageIndex) Images() []regv1.Image {
return i.images
}

// Indexes retrieve all indexes associated with the described image
func (i DescribedImageIndex) Indexes() []regv1.ImageIndex {
return i.indexes
}

// ImageIndex retrieve the index that matches the provided digest
func (i DescribedImageIndex) ImageIndex(digest regv1.Hash) (regv1.ImageIndex, error) {
for _, idx := range i.indexes {
idxDigest, err := idx.Digest()
Expand Down

0 comments on commit c6fae29

Please sign in to comment.