Skip to content

Commit

Permalink
Merge pull request #3863 from deitch/rationalize-build-pull
Browse files Browse the repository at this point in the history
Rationalize build pull
  • Loading branch information
deitch committed Nov 7, 2022
2 parents bfa4a33 + a3995bb commit 860163a
Show file tree
Hide file tree
Showing 5 changed files with 152 additions and 19 deletions.
57 changes: 57 additions & 0 deletions src/cmd/linuxkit/cache/write.go
Original file line number Diff line number Diff line change
Expand Up @@ -353,3 +353,60 @@ func (p *Provider) DescriptorWrite(ref *reference.Spec, desc v1.Descriptor) (lkt
&desc,
), nil
}

func (p *Provider) ImageInCache(ref *reference.Spec, trustedRef, architecture string) (bool, error) {
if _, err := p.findImage(ref.String(), architecture); err != nil {
return false, err
}
return true, nil
}

// ImageInRegistry takes an image name and checks that the image manifest or index to which it refers
// exists in the registry.
func (p *Provider) ImageInRegistry(ref *reference.Spec, trustedRef, architecture string) (bool, error) {
image := ref.String()
remoteOptions := []remote.Option{remote.WithAuthFromKeychain(authn.DefaultKeychain)}
log.Debugf("Checking image %s in registry", image)

remoteRef, err := name.ParseReference(image)
if err != nil {
return false, fmt.Errorf("invalid image name %s: %v", image, err)
}

desc, err := remote.Get(remoteRef, remoteOptions...)
if err != nil {
return false, fmt.Errorf("error getting manifest for image %s: %v", image, err)
}
// first attempt as an index
ii, err := desc.ImageIndex()
if err == nil {
log.Debugf("ImageExists retrieved %s as index", remoteRef)
im, err := ii.IndexManifest()
if err != nil {
return false, fmt.Errorf("unable to get IndexManifest: %v", err)
}
for _, m := range im.Manifests {
if m.MediaType.IsImage() && (m.Platform == nil || m.Platform.Architecture == architecture) {
return true, nil
}
}
// we went through all of the manifests and did not find one that matches the target architecture
} else {
var im v1.Image
// try an image
im, err = desc.Image()
if err != nil {
return false, fmt.Errorf("provided image is neither an image nor an index: %s", image)
}
log.Debugf("ImageExists retrieved %s as image", remoteRef)
conf, err := im.ConfigFile()
if err != nil {
return false, fmt.Errorf("unable to get ConfigFile: %v", err)
}
if conf.Architecture == architecture {
return true, nil
}
// the image had the wrong architecture
}
return false, nil
}
14 changes: 12 additions & 2 deletions src/cmd/linuxkit/pkg_build.go
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,7 @@ func pkgBuildPush(args []string, withPush bool) {
}

force := flags.Bool("force", false, "Force rebuild even if image is in local cache")
pull := flags.Bool("pull", false, "Pull image if in registry but not in local cache; conflicts with --force")
ignoreCache := flags.Bool("ignore-cached", false, "Ignore cached intermediate images, always pulling from registry")
docker := flags.Bool("docker", false, "Store the built image in the docker image cache instead of the default linuxkit cache")
platforms := flags.String("platforms", "", "Which platforms to build for, defaults to all of those for which the package can be built")
Expand All @@ -49,8 +50,10 @@ func pkgBuildPush(args []string, withPush bool) {
flags.Var(&cacheDir, "cache", fmt.Sprintf("Directory for caching and finding cached image, overrides env var %s", envVarCacheDir))

// some logic clarification:
// pkg build - always builds unless is in cache
// pkg build --force - always builds even if is in cache
// pkg build - builds unless is in cache or published in registry
// pkg build --pull - builds unless is in cache or published in registry; pulls from registry if not in cache
// pkg build --force - always builds even if is in cache or published in registry
// pkg build --force --pull - always builds even if is in cache or published in registry; --pull ignored
// pkg push - always builds unless is in cache
// pkg push --force - always builds even if is in cache
// pkg push --nobuild - skips build; if not in cache, fails
Expand Down Expand Up @@ -78,6 +81,10 @@ func pkgBuildPush(args []string, withPush bool) {
fmt.Fprint(os.Stderr, "flags -force and -nobuild conflict")
os.Exit(1)
}
if *pull && *force {
fmt.Fprint(os.Stderr, "flags -force and -pull conflict")
os.Exit(1)
}

var opts []pkglib.BuildOpt
if *force {
Expand All @@ -86,6 +93,9 @@ func pkgBuildPush(args []string, withPush bool) {
if *ignoreCache {
opts = append(opts, pkglib.WithBuildIgnoreCache())
}
if *pull {
opts = append(opts, pkglib.WithBuildPull())
}

opts = append(opts, pkglib.WithBuildCacheDir(cacheDir.String()))

Expand Down
76 changes: 59 additions & 17 deletions src/cmd/linuxkit/pkglib/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
type buildOpts struct {
skipBuild bool
force bool
pull bool
ignoreCache bool
push bool
release string
Expand Down Expand Up @@ -61,6 +62,14 @@ func WithBuildForce() BuildOpt {
}
}

// WithBuildPull pull down the image to cache if it already exists in registry
func WithBuildPull() BuildOpt {
return func(bo *buildOpts) error {
bo.pull = true
return nil
}
}

// WithBuildPush pushes the result of the build to the registry
func WithBuildPush() BuildOpt {
return func(bo *buildOpts) error {
Expand Down Expand Up @@ -89,6 +98,7 @@ func WithRelease(r string) BuildOpt {
func WithBuildTargetDockerCache() BuildOpt {
return func(bo *buildOpts) error {
bo.targetDocker = true
bo.pull = true // if we are to load it into docker, it must be in local cache
return nil
}
}
Expand Down Expand Up @@ -225,29 +235,61 @@ func (p Pkg) Build(bos ...BuildOpt) error {
}

var platformsToBuild []imagespec.Platform
if bo.force {
switch {
case bo.force && bo.skipBuild:
return errors.New("cannot force build and skip build")
case bo.force:
// force local build
platformsToBuild = bo.platforms
} else if !bo.skipBuild {
fmt.Fprintf(writer, "checking for %s in local cache, fallback to remote registry...\n", ref)
case bo.skipBuild:
// do not build anything if we explicitly did skipBuild
platformsToBuild = nil
default:
// check local cache, fallback to check registry / pull image from registry, fallback to build
fmt.Fprintf(writer, "checking for %s in local cache...\n", ref)
for _, platform := range bo.platforms {
if _, err := c.ImagePull(&ref, "", platform.Architecture, false); err == nil {
fmt.Fprintf(writer, "%s found or pulled\n", ref)
if bo.targetDocker {
archRef, err := reference.Parse(fmt.Sprintf("%s-%s", p.FullTag(), platform.Architecture))
if err != nil {
return err
}
fmt.Fprintf(writer, "checking for %s in local cache, fallback to remote registry...\n", archRef)
if _, err := c.ImagePull(&archRef, "", platform.Architecture, false); err == nil {
fmt.Fprintf(writer, "%s found or pulled\n", archRef)
} else {
fmt.Fprintf(writer, "%s not found, will build: %s\n", archRef, err)
platformsToBuild = append(platformsToBuild, platform)
if exists, err := c.ImageInCache(&ref, "", platform.Architecture); err == nil && exists {
fmt.Fprintf(writer, "found %s in local cache, skipping build\n", ref)
continue
}
if bo.pull {
// need to pull the image from the registry, else build
fmt.Fprintf(writer, "%s %s not found in local cache, trying to pull\n", ref, platform.Architecture)
if _, err := c.ImagePull(&ref, "", platform.Architecture, false); err == nil {
fmt.Fprintf(writer, "%s pulled\n", ref)
if bo.targetDocker {
archRef, err := reference.Parse(fmt.Sprintf("%s-%s", p.FullTag(), platform.Architecture))
if err != nil {
return err
}
fmt.Fprintf(writer, "checking for %s in local cache, fallback to remote registry...\n", archRef)
if _, err := c.ImagePull(&archRef, "", platform.Architecture, false); err == nil {
fmt.Fprintf(writer, "%s found or pulled\n", archRef)
} else {
fmt.Fprintf(writer, "%s not found, will build: %s\n", archRef, err)
platformsToBuild = append(platformsToBuild, platform)
}
}
// successfully pulled, no need to build, continue with next platform
continue
}
} else {
fmt.Fprintf(writer, "%s not found, will build: %s\n", ref, err)
platformsToBuild = append(platformsToBuild, platform)
} else {
// do not pull, just check if it exists in a registry
fmt.Fprintf(writer, "%s %s not found in local cache, checking registry\n", ref, platform.Architecture)
exists, err := c.ImageInRegistry(&ref, "", platform.Architecture)
if err != nil {
return fmt.Errorf("error checking remote registry for %s: %v", ref, err)
}

if exists {
fmt.Fprintf(writer, "%s %s found on registry\n", ref, platform.Architecture)
continue
}
fmt.Fprintf(writer, "%s %s not found, will build\n", ref, platform.Architecture)
platformsToBuild = append(platformsToBuild, platform)

}
}
}
Expand Down
18 changes: 18 additions & 0 deletions src/cmd/linuxkit/pkglib/build_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,24 @@ func (c *cacheMocker) ImagePull(ref *reference.Spec, trustedRef, architecture st
return c.imageWriteStream(ref, architecture, bytes.NewReader(b))
}

func (c *cacheMocker) ImageInCache(ref *reference.Spec, trustedRef, architecture string) (bool, error) {
image := ref.String()
desc, ok := c.images[image]
if !ok {
return false, nil
}
for _, d := range desc {
if d.Platform != nil && d.Platform.Architecture == architecture {
return true, nil
}
}
return false, nil
}

func (c *cacheMocker) ImageInRegistry(ref *reference.Spec, trustedRef, architecture string) (bool, error) {
return false, nil
}

func (c *cacheMocker) ImageLoad(ref *reference.Spec, architecture string, r io.Reader) (lktspec.ImageSource, error) {
if !c.enableImageLoad {
return nil, errors.New("ImageLoad disabled")
Expand Down
6 changes: 6 additions & 0 deletions src/cmd/linuxkit/spec/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,12 @@ type CacheProvider interface {
// efficient and only write missing blobs, based on their content hash. If the ref already
// exists in the cache, it should not pull anything, unless alwaysPull is set to true.
ImagePull(ref *reference.Spec, trustedRef, architecture string, alwaysPull bool) (ImageSource, error)
// ImageInCache takes an image name and checks if it exists in the cache, including checking that the given
// architecture is complete. Like ImagePull, it should be efficient and only write missing blobs, based on
// their content hash.
ImageInCache(ref *reference.Spec, trustedRef, architecture string) (bool, error)
// ImageInRegistry takes an image name and checks if it exists in the registry.
ImageInRegistry(ref *reference.Spec, trustedRef, architecture string) (bool, error)
// IndexWrite takes an image name and creates an index for the descriptors to which it points.
// Cache implementation determines whether it should pull missing blobs from a remote registry.
// If the provided reference already exists and it is an index, updates the manifests in the
Expand Down

0 comments on commit 860163a

Please sign in to comment.