-
Notifications
You must be signed in to change notification settings - Fork 30
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Sketch code for downloading images to cache
Rough-and-ready code for downloading images into the cache. Manifests referred to by digest get written out; otherwise, they get tagged.
- Loading branch information
Showing
8 changed files
with
631 additions
and
20 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,169 @@ | ||
| package cache | ||
|
|
||
| // The read part of the cache constructs a filesystem given an image | ||
| // ref. The download part, here, gets the constituents of images and | ||
| // puts them into the cache. The contract between the two is in the | ||
| // layout of the directories used by the cache, described in the | ||
| // package documentation. | ||
|
|
||
| import ( | ||
| "archive/tar" | ||
| "fmt" | ||
| "io" | ||
| "io/ioutil" | ||
| "os" | ||
| "path/filepath" | ||
|
|
||
| "github.com/google/go-containerregistry/pkg/name" | ||
| "github.com/google/go-containerregistry/pkg/v1/remote" | ||
| ) | ||
|
|
||
| // FIXME don't force permissions on the actual files, since some may | ||
| // need to be executable | ||
| const cacheFileMode = os.FileMode(0400) | ||
| const cacheDirMode = os.FileMode(0700) | ||
|
|
||
| // Download makes sure the manifest and layers for a particular image | ||
| // are present in the cache. | ||
| func (c *Cache) Download(image string) error { // <-- could return the digest? | ||
| // Ref for this code: | ||
| // https://github.com/google/go-containerregistry/blob/master/pkg/crane/pull.go#Save | ||
| ref, err := name.ParseReference(image) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| manifestPath := c.manifestPath(ref) | ||
| if manifestPath == "" { | ||
| return fmt.Errorf("cannot make manifest path for image ref %q", ref) | ||
| } | ||
|
|
||
| // Whichever kind of image ref, if the manifest is already in the | ||
| // filesystem, we must have completed this previously. | ||
| _, err = os.Stat(manifestPath) | ||
| if err == nil { | ||
| return nil | ||
| } | ||
| if !os.IsNotExist(err) { | ||
| return err | ||
| } | ||
|
|
||
| img, err := remote.Image(ref) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // If it's a reference with a tag, specifically, then look for the | ||
| // manifest at the digest as well | ||
| var tagManifestPath string | ||
| if _, ok := ref.(name.Tag); ok { | ||
| dig, err := img.Digest() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| digestManifestPath := c.manifestPath(ref.Context().Digest(dig.String())) | ||
| if digestManifestPath == "" { | ||
| return fmt.Errorf("unable to construct path to manifest for %q", dig) | ||
| } | ||
| // we'll be writing to the digest path and symlinking the tag | ||
| // path to the digest path | ||
| tagManifestPath, manifestPath = manifestPath, digestManifestPath | ||
| _, err = os.Stat(manifestPath) | ||
| if err == nil { | ||
| // TODO(michael): factor this out | ||
| if err = os.MkdirAll(filepath.Dir(tagManifestPath), cacheDirMode); err != nil { | ||
| return err | ||
| } | ||
| err = os.Symlink(manifestPath, tagManifestPath) | ||
| return err | ||
| } | ||
| if !os.IsNotExist(err) { | ||
| return err | ||
| } | ||
| } | ||
|
|
||
| // TODO any kind of verification | ||
|
|
||
| layers, err := img.Layers() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| for _, layer := range layers { | ||
| // Check for the layer in the layers directory | ||
| digest, err := layer.Digest() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| p := c.layerPath(digest.Algorithm, digest.Hex) | ||
| _, err = os.Stat(p) | ||
| if err == nil { // already have it | ||
| continue | ||
| } | ||
|
|
||
| // TODO(michael): if there's a problem after this, clean up | ||
| // the expanded layer before returning, so it doesn't look | ||
| // like we've succeeded. In fact, better to expand somewhere | ||
| // else, then rename it to the right place if it succeeds. | ||
| if err = os.MkdirAll(p, cacheDirMode); err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // Write the layer by expanding it into the directory. | ||
|
|
||
| layerReader, err := layer.Uncompressed() | ||
| if err != nil { | ||
| return err | ||
| } | ||
| rdr := tar.NewReader(layerReader) | ||
| for { | ||
| hdr, err := rdr.Next() | ||
| if err == io.EOF { | ||
| break | ||
| } | ||
| targetPath := filepath.Join(p, hdr.Name) | ||
| info := hdr.FileInfo() | ||
| if info.IsDir() { | ||
| if os.MkdirAll(targetPath, 0755); err != nil { | ||
| // TODO cleanup | ||
| return err | ||
| } | ||
| } else { | ||
| f, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR, cacheFileMode) | ||
| if err != nil { | ||
| return err // TODO cleanup | ||
| } | ||
| if _, err := io.Copy(f, rdr); err != nil { | ||
| f.Close() | ||
| return err // TODO cleanup | ||
| } | ||
| f.Close() | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // TODO(michael) figure out if we will always get an OCI v1 | ||
| // (compatible) manifest | ||
| man, err := img.RawManifest() | ||
| if err != nil { | ||
| // TODO cleanup | ||
| return err | ||
| } | ||
|
|
||
| if err = os.MkdirAll(filepath.Dir(manifestPath), cacheDirMode); err != nil { | ||
| return err | ||
| } | ||
| if err = ioutil.WriteFile(manifestPath, man, cacheFileMode); err != nil { | ||
| // TODO cleanup | ||
| return err | ||
| } | ||
|
|
||
| if tagManifestPath != "" { | ||
| if err = os.MkdirAll(filepath.Dir(tagManifestPath), cacheDirMode); err != nil { | ||
| return err | ||
| } | ||
| err = os.Symlink(manifestPath, tagManifestPath) | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,45 @@ | ||
| package cache | ||
|
|
||
| import ( | ||
| "io/ioutil" | ||
| "net/http/httptest" | ||
| "os" | ||
| "testing" | ||
|
|
||
| "github.com/google/go-containerregistry/pkg/crane" | ||
| "github.com/google/go-containerregistry/pkg/registry" | ||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func setupRegistry(t *testing.T) (*httptest.Server, string) { | ||
| regHandler := registry.New() | ||
| regSrv := httptest.NewServer(regHandler) | ||
| img, err := crane.Load("./testfiles/helloworld.tar") | ||
| assert.NoError(t, err) | ||
| newImg := regSrv.URL[len("http://"):] + "/helloworld:linux" | ||
| assert.NoError(t, crane.Push(img, newImg)) | ||
| return regSrv, newImg | ||
| } | ||
|
|
||
| func TestDownloadToCache(t *testing.T) { | ||
| tmp, err := ioutil.TempDir("", "jk-test") | ||
| assert.NoError(t, err) | ||
| defer os.RemoveAll(tmp) | ||
|
|
||
| cache := New(tmp) | ||
|
|
||
| regSrv, img := setupRegistry(t) | ||
| defer regSrv.Close() | ||
|
|
||
| err = cache.Download(img) | ||
| assert.NoError(t, err) | ||
|
|
||
| ov, err := cache.FileSystemForImage(mustParseRef(img)) | ||
| assert.NoError(t, err) | ||
| f, err := ov.Open("/hello") | ||
| assert.NoError(t, err) | ||
| defer f.Close() | ||
|
|
||
| _, err = ioutil.ReadAll(f) | ||
| assert.NoError(t, err) | ||
| } |
File renamed without changes.
Binary file not shown.