Use image references as just bein pushed to populate bundle

As we push images and retrieve Digests, we don't need fixup bundle to
check them presence on registry

Signed-off-by: Nicolas De Loof <nicolas.deloof@gmail.com>

e2e/commands_test.go

@@ -75,6 +75,9 @@ func TestRenderFormatters(t *testing.T) {
 	defer cleanup()
 
 	appPath := filepath.Join("testdata", "simple", "simple.dockerapp")
+	cmd.Command = dockerCli.Command("app", "build", appPath)
+	icmd.RunCmd(cmd).Assert(t, icmd.Success)
+
 	cmd.Command = dockerCli.Command("app", "render", "--formatter", "json", appPath)
 	result := icmd.RunCmd(cmd).Assert(t, icmd.Success)
 	golden.Assert(t, result.Stdout(), "expected-json-render.golden")

e2e/helper_test.go

@@ -43,10 +43,11 @@ func runWithDindSwarmAndRegistry(t *testing.T, todo func(dindSwarmAndRegistryInf
 	// Solution found is: fix the port of the registry to be the same internally and externally
 	// and run the dind container in the same network namespace: this way 127.0.0.1:<registry-port> both resolves to the registry from the client and from dind
 
-	swarm := NewContainer("docker:18.09-dind", 2375, "--insecure-registry", fmt.Sprintf("127.0.0.1:%d", registryPort))
+	swarm := NewContainer("docker:19.03.2-dind", 2375, "--insecure-registry", fmt.Sprintf("127.0.0.1:%d", registryPort))
 	swarm.Start(t, "--expose", strconv.FormatInt(int64(registryPort), 10),
 		"-p", fmt.Sprintf("%d:%d", registryPort, registryPort),
-		"-p", "2375")
+		"-p", "2375",
+		"-e", "DOCKER_TLS_CERTDIR=") // Disable certificate generate on DinD startup
 	defer swarm.Stop(t)
 
 	registry := NewContainer("registry:2", registryPort)
@@ -76,6 +77,10 @@ func runWithDindSwarmAndRegistry(t *testing.T, todo func(dindSwarmAndRegistryInf
 	cmd.Command = dockerCli.Command("load", "-i", tmpDir.Join("cnab-app-base.tar.gz"))
 	icmd.RunCmd(cmd).Assert(t, icmd.Success)
 
+	// Compose files used for e2e test rely on busybox:1.30.1 image being pushed. Let's pretend we have it pulled allready
+	cmd.Command = dockerCli.Command("tag", fmt.Sprintf("docker/cnab-app-base:%s", internal.Version), "busybox:1.30.1")
+	icmd.RunCmd(cmd).Assert(t, icmd.Success)
+
 	info := dindSwarmAndRegistryInfo{
 		configuredCmd:   cmd,
 		registryAddress: registry.GetAddress(t),

e2e/images_test.go

@@ -22,10 +22,13 @@ b-simple-app           latest simple
 
 func insertBundles(t *testing.T, cmd icmd.Cmd, dir *fs.Dir, info dindSwarmAndRegistryInfo) string {
 	// Push an application so that we can later pull it by digest
-	cmd.Command = dockerCli.Command("app", "push", "--tag", info.registryAddress+"/c-myapp", filepath.Join("testdata", "push-pull", "push-pull.dockerapp"))
+	cmd.Command = dockerCli.Command("app", "build", "--tag", info.registryAddress+"/c-myapp", filepath.Join("testdata", "push-pull", "push-pull.dockerapp"))
 	r := icmd.RunCmd(cmd).Assert(t, icmd.Success)
 
-	// Get the digest from the output of the pull command
+	cmd.Command = dockerCli.Command("app", "push", info.registryAddress+"/c-myapp")
+	icmd.RunCmd(cmd).Assert(t, icmd.Success)
+
+	// Get the digest from the output of the push command
 	out := r.Stdout()
 	matches := reg.FindAllStringSubmatch(out, 1)
 	digest := matches[0][1]
@@ -34,9 +37,9 @@ func insertBundles(t *testing.T, cmd icmd.Cmd, dir *fs.Dir, info dindSwarmAndReg
 	cmd.Command = dockerCli.Command("app", "pull", info.registryAddress+"/c-myapp@"+digest)
 	icmd.RunCmd(cmd).Assert(t, icmd.Success)
 
-	cmd.Command = dockerCli.Command("app", "bundle", filepath.Join("testdata", "simple", "simple.dockerapp"), "--tag", "b-simple-app", "--output", dir.Join("simple-bundle.json"))
+	cmd.Command = dockerCli.Command("app", "build", filepath.Join("testdata", "simple", "simple.dockerapp"), "--tag", "b-simple-app")
 	icmd.RunCmd(cmd).Assert(t, icmd.Success)
-	cmd.Command = dockerCli.Command("app", "bundle", filepath.Join("testdata", "simple", "simple.dockerapp"), "--tag", "a-simple-app", "--output", dir.Join("simple-bundle.json"))
+	cmd.Command = dockerCli.Command("app", "build", filepath.Join("testdata", "simple", "simple.dockerapp"), "--tag", "a-simple-app")
 	icmd.RunCmd(cmd).Assert(t, icmd.Success)
 
 	return digest
@@ -66,10 +69,23 @@ func TestImageRm(t *testing.T) {
 		digest := insertBundles(t, cmd, dir, info)
 
 		cmd.Command = dockerCli.Command("app", "image", "rm", info.registryAddress+"/c-myapp@"+digest)
-		icmd.RunCmd(cmd).Assert(t, icmd.Success)
-
-		cmd.Command = dockerCli.Command("app", "image", "rm", "a-simple-app:latest", "b-simple-app:latest")
-		icmd.RunCmd(cmd).Assert(t, icmd.Success)
+		icmd.RunCmd(cmd).Assert(t, icmd.Expected{
+			ExitCode: 0,
+			Out:      "Deleted: " + info.registryAddress + "/c-myapp@" + digest,
+		})
+
+		cmd.Command = dockerCli.Command("app", "image", "rm", "a-simple-app", "b-simple-app:latest")
+		icmd.RunCmd(cmd).Assert(t, icmd.Expected{
+			ExitCode: 0,
+			Out: `Deleted: a-simple-app:latest
+Deleted: b-simple-app:latest`,
+		})
+
+		cmd.Command = dockerCli.Command("app", "image", "rm", "b-simple-app")
+		icmd.RunCmd(cmd).Assert(t, icmd.Expected{
+			ExitCode: 1,
+			Err:      `Error: no such image b-simple-app:latest`,
+		})
 
 		expectedOutput := "REPOSITORY TAG APP NAME\n"
 		cmd.Command = dockerCli.Command("app", "image", "ls")

e2e/pushpull_test.go

@@ -95,12 +95,16 @@ func TestPushArchs(t *testing.T) {
 			t.Run(testCase.name, func(t *testing.T) {
 				cmd := info.configuredCmd
 				ref := info.registryAddress + "/test/push-pull:1"
-				args := []string{"app", "push", "--tag", ref}
+
+				args := []string{"app", "build", "--tag", ref}
 				args = append(args, testCase.args...)
 				args = append(args, filepath.Join("testdata", "push-pull", "push-pull.dockerapp"))
 				cmd.Command = dockerCli.Command(args...)
 				icmd.RunCmd(cmd).Assert(t, icmd.Success)
 
+				cmd.Command = dockerCli.Command("app", "push", ref)
+				icmd.RunCmd(cmd).Assert(t, icmd.Success)
+
 				var index v1.Index
 				headers := map[string]string{
 					"Accept": "application/vnd.docker.distribution.manifest.list.v2+json",
@@ -128,11 +132,15 @@ func TestPushInsecureRegistry(t *testing.T) {
 		// create a command outside of the dind context so without the insecure registry configured
 		cmd2, cleanup2 := dockerCli.createTestCmd()
 		defer cleanup2()
-		cmd2.Command = dockerCli.Command("app", "push", "--tag", ref, filepath.Join("testdata", "push-pull", "push-pull.dockerapp"))
+		cmd2.Command = dockerCli.Command("app", "build", "--tag", ref, filepath.Join("testdata", "push-pull", "push-pull.dockerapp"))
+		icmd.RunCmd(cmd2).Assert(t, icmd.Success)
+		cmd2.Command = dockerCli.Command("app", "push", ref)
 		icmd.RunCmd(cmd2).Assert(t, icmd.Expected{ExitCode: 1})
 
 		// run the push with the command inside dind context configured to allow access to the insecure registry
 		cmd := info.configuredCmd
+		cmd2.Command = dockerCli.Command("app", "build", "--tag", ref, filepath.Join("testdata", "push-pull", "push-pull.dockerapp"))
+		icmd.RunCmd(cmd2).Assert(t, icmd.Success)
 		cmd.Command = dockerCli.Command("app", "push", "--tag", ref, filepath.Join("testdata", "push-pull", "push-pull.dockerapp"))
 		icmd.RunCmd(cmd).Assert(t, icmd.Success)
 	})
@@ -142,7 +150,9 @@ func TestPushInstall(t *testing.T) {
 	runWithDindSwarmAndRegistry(t, func(info dindSwarmAndRegistryInfo) {
 		cmd := info.configuredCmd
 		ref := info.registryAddress + "/test/push-pull"
-		cmd.Command = dockerCli.Command("app", "push", "--tag", ref, filepath.Join("testdata", "push-pull", "push-pull.dockerapp"))
+		cmd.Command = dockerCli.Command("app", "build", "--tag", ref, filepath.Join("testdata", "push-pull", "push-pull.dockerapp"))
+		icmd.RunCmd(cmd).Assert(t, icmd.Success)
+		cmd.Command = dockerCli.Command("app", "push", ref)
 		icmd.RunCmd(cmd).Assert(t, icmd.Success)
 
 		cmd.Command = dockerCli.Command("app", "install", ref, "--name", t.Name())
@@ -157,7 +167,11 @@ func TestPushPullInstall(t *testing.T) {
 		cmd := info.configuredCmd
 		ref := info.registryAddress + "/test/push-pull"
 		tag := ":v.0.0.1"
-		cmd.Command = dockerCli.Command("app", "push", "--tag", ref+tag, filepath.Join("testdata", "push-pull", "push-pull.dockerapp"))
+
+		cmd.Command = dockerCli.Command("app", "build", "--tag", ref+tag, filepath.Join("testdata", "push-pull", "push-pull.dockerapp"))
+		icmd.RunCmd(cmd).Assert(t, icmd.Success)
+
+		cmd.Command = dockerCli.Command("app", "push", ref+tag)
 		icmd.RunCmd(cmd).Assert(t, icmd.Success)
 		cmd.Command = dockerCli.Command("app", "pull", ref+tag)
 		icmd.RunCmd(cmd).Assert(t, icmd.Success)
@@ -193,8 +207,10 @@ func TestPushInstallBundle(t *testing.T) {
 		defer tmpDir.Remove()
 		bundleFile := tmpDir.Join("bundle.json")
 
-		// render the app to a bundle, we use the app from the push pull test above.
-		cmd.Command = dockerCli.Command("app", "bundle", "-o", bundleFile, filepath.Join("testdata", "push-pull", "push-pull.dockerapp"))
+		cmd.Command = dockerCli.Command("app", "build", "-o", bundleFile, filepath.Join("testdata", "push-pull", "push-pull.dockerapp"))
+		icmd.RunCmd(cmd).Assert(t, icmd.Success)
+
+		cmd.Command = dockerCli.Command("pull", "busybox:1.30.1")
 		icmd.RunCmd(cmd).Assert(t, icmd.Success)
 
 		// push it and install to check it is available
@@ -220,7 +236,7 @@ func TestPushInstallBundle(t *testing.T) {
 		t.Run("push-ref", func(t *testing.T) {
 			name := strings.Replace(t.Name(), "/", "_", 1)
 			ref2 := info.registryAddress + "/test/push-ref"
-			cmd.Command = dockerCli.Command("app", "push", "--tag", ref2, ref+":latest")
+			cmd.Command = dockerCli.Command("app", "push", refgit2)
 			icmd.RunCmd(cmd).Assert(t, icmd.Success)
 
 			cmd.Command = dockerCli.Command("app", "install", ref2, "--name", name)
@@ -242,7 +258,7 @@ func TestPushInstallBundle(t *testing.T) {
 			cmd2.Env = append(cmd2.Env, "DOCKER_CONTEXT=swarm-context")
 
 			// bundle the app again but this time with a tag to store it into the bundle store
-			cmd2.Command = dockerCli.Command("app", "bundle", "--tag", ref2, "-o", bundleFile, filepath.Join("testdata", "push-pull", "push-pull.dockerapp"))
+			cmd2.Command = dockerCli.Command("app", "build", "--tag", ref2, "-o", bundleFile, filepath.Join("testdata", "push-pull", "push-pull.dockerapp"))
 			icmd.RunCmd(cmd2).Assert(t, icmd.Success)
 			// Push the app without tagging it explicitly
 			cmd2.Command = dockerCli.Command("app", "push", ref2)

internal/commands/build.go

@@ -59,7 +59,7 @@ func buildCmd(dockerCli command.Cli) *cobra.Command {
 	flags.BoolVar(&opts.noCache, "no-cache", false, "Do not use cache when building the image")
 	flags.StringVar(&opts.progress, "progress", "auto", "Set type of progress output (auto, plain, tty). Use plain to show container output")
 	flags.BoolVar(&opts.pull, "pull", false, "Always attempt to pull a newer version of the image")
-	flags.StringVarP(&opts.out, "out", "o", "", "Dump generated bundle into a file")
+	flags.StringVarP(&opts.out, "output", "o", "", "Dump generated bundle into a file")
 	flags.StringVarP(&opts.tag, "tag", "t", "", "Name and optionally a tag in the 'name:tag' format")
 
 	return cmd
@@ -79,6 +79,7 @@ func runBuild(dockerCli command.Cli, application string, opt buildOptions) (refe
 	}
 
 	ctx := appcontext.Context()
+
 	compose, err := bake.ParseCompose(app.Composes()[0]) // Fixme can have > 1 composes ?
 	if err != nil {
 		return nil, err
@@ -109,7 +110,7 @@ func runBuild(dockerCli command.Cli, application string, opt buildOptions) (refe
 	}
 
 	if logrus.IsLevelEnabled(logrus.DebugLevel)	{
-		dt, err := json.MarshalIndent(map[string]map[string]bake.Target{"target": targets}, "", "   ")
+		dt, err := json.MarshalIndent(targets, "", "   ")
 		if err != nil {
 			return nil, err
 		}
@@ -155,6 +156,13 @@ func runBuild(dockerCli command.Cli, application string, opt buildOptions) (refe
 	}
 
 	fmt.Println("Successfully built service images")
+	if logrus.IsLevelEnabled(logrus.DebugLevel)	{
+		dt, err := json.MarshalIndent(resp, "", "   ")
+		if err != nil {
+			return nil, err
+		}
+		logrus.Debug(string(dt))
+	}
 
 	for service, r := range resp {
 		digest := r.ExporterResponse["containerimage.digest"]

internal/commands/bundle.go

@@ -1,6 +1,8 @@
 package commands
 
 import (
+	"bytes"
+	"context"
 	"fmt"
 	"github.com/deislabs/cnab-go/bundle"
 	"github.com/docker/app/internal/packager"
@@ -10,8 +12,11 @@ import (
 	"github.com/docker/cli/cli/command"
 	"github.com/docker/cli/cli/config"
 	"github.com/docker/distribution/reference"
+	dockertypes "github.com/docker/docker/api/types"
+	"github.com/docker/docker/pkg/jsonmessage"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
+	"io/ioutil"
 )
 
 func makeBundleFromApp(dockerCli command.Cli, app *types.App, refOverride reference.NamedTagged) (*bundle.Bundle, error) {
@@ -22,6 +27,31 @@ func makeBundleFromApp(dockerCli command.Cli, app *types.App, refOverride refere
 		return nil, err
 	}
 
+	buildContext := bytes.NewBuffer(nil)
+	if err := packager.PackInvocationImageContext(dockerCli, app, buildContext); err != nil {
+			return nil, err
+		}
+
+	logrus.Debugf("Building invocation image %s", invocationImageName)
+	buildResp, err := dockerCli.Client().ImageBuild(context.TODO(), buildContext, dockertypes.ImageBuildOptions{
+			Dockerfile: "Dockerfile",
+			Tags:       []string{invocationImageName},
+			BuildArgs:  map[string]*string{},
+		})
+	if err != nil {
+		return nil, err
+	}
+	defer buildResp.Body.Close()
+
+	if err := jsonmessage.DisplayJSONMessagesStream(buildResp.Body, ioutil.Discard, 0, false, func(jsonmessage.JSONMessage) {}); err != nil {
+		// If the invocation image can't be found we will get an error of the form:
+		// manifest for docker/cnab-app-base:v0.6.0-202-gbaf0b246c7 not found
+		if err.Error() == fmt.Sprintf("manifest for %s not found", packager.BaseInvocationImage(dockerCli)) {
+			return nil, fmt.Errorf("unable to resolve Docker App base image: %s", packager.BaseInvocationImage(dockerCli))
+		}
+		return nil, err
+	}
+
 	return packager.ToCNAB(app, invocationImageName)
 }
 

internal/commands/push.go

@@ -3,7 +3,10 @@ package commands
 import (
 	"bytes"
 	"context"
+	"encoding/json"
 	"fmt"
+	"github.com/docker/cli/cli/streams"
+	"github.com/docker/distribution/manifest/schema2"
 	"io"
 	"os"
 	"strings"
@@ -73,6 +76,7 @@ func runPush(dockerCli command.Cli, ref string, opts pushOptions) error {
 		return err
 	}
 	named = reference.TagNameOnly(named)
+	repo := fmt.Sprintf("%s/%s", reference.Domain(named), reference.Path(named))
 
 	// Get the bundle
 	bundleStore, err := prepareBundleStore()
@@ -92,7 +96,8 @@ func runPush(dockerCli command.Cli, ref string, opts pushOptions) error {
 		if err != nil {
 			return err
 		}
-		image.Image = fmt.Sprintf("%s@%s", bndl.Name, digest.String())
+		image.Image = fmt.Sprintf("%s@%s", repo, digest.String())
+		image.MediaType = schema2.MediaTypeManifest
 		bndl.Images[service] = image
 	}
 
@@ -103,10 +108,13 @@ func runPush(dockerCli command.Cli, ref string, opts pushOptions) error {
 		if err != nil {
 			return err
 		}
-		bndl.InvocationImages[i].Image = fmt.Sprintf("%s@%s", bndl.Name, digest.String())
+		image.MediaType = schema2.MediaTypeManifest
+		image.Image = fmt.Sprintf("%s@%s", repo, digest.String())
+		bndl.InvocationImages[i] = image
 	}
 
 	// Push the bundle
+	fmt.Fprintf(dockerCli.Out(), "Pushing bundle\n")
 	if err = pushBundle(dockerCli, opts, bndl, named); err != nil {
 		return err
 	}
@@ -151,16 +159,52 @@ func pushImage(dockerCli command.Cli, opts pushOptions, ref reference.Named) (di
 		return "", errors.Wrapf(err, "starting push of %q", ref.String())
 	}
 	defer reader.Close()
-	if err := jsonmessage.DisplayJSONMessagesToStream(reader, dockerCli.Out(), nil); err != nil {
+	d := digestCollector{out:dockerCli.Out()}
+	if err := jsonmessage.DisplayJSONMessagesToStream(reader, &d, nil); err != nil {
 		return "", errors.Wrapf(err, "pushing to %q", ref.String())
 	}
 
-	resolver := remotes.CreateResolver(dockerCli.ConfigFile(), opts.registry.insecureRegistries...)
-	_, desc, err := resolver.Resolve(context.TODO(), ref.String())
+	// First attempt : retrieve the registry digest from push stdout
+	// FIXME wonder there's a better way, or maybe we could reuse some existing code for this purpose
+	dg, err := d.Digest()
+	if err == nil && dg != "" {
+		return dg, nil
+	}
+
+	// Second attempt: query registry for the tag we just pushed
+	// FIXME potential race condition
+	t, err := dockerCli.RegistryClient(false).GetManifest(context.TODO(), ref)
 	if err != nil {
-		return "", err
+		return "", errors.Wrapf(err, "failed to retrieve image digest %s", ref.String())
 	}
-	return desc.Digest, nil
+	return t.Descriptor.Digest, nil
+}
+
+type digestCollector struct {
+	out *streams.Out
+	last string
+}
+
+// Write implement writer.Write
+func (d *digestCollector) Write(p []byte) (n int, err error) {
+	d.last = string(p)
+	return d.out.Write(p)
+}
+
+// Digest return the image digest collected by parsing "docker push" stdout
+func (d digestCollector) Digest() (digest.Digest, error) {
+	dg := digest.DigestRegexp.FindString(d.last)
+	return digest.Parse(dg)
+}
+
+// FD implement stream.FD
+func (d *digestCollector) FD() uintptr {
+	return d.out.FD()
+}
+
+// IsTerminal implement stream.IsTerminal
+func (d *digestCollector) IsTerminal() bool {
+	return d.out.IsTerminal()
 }
 
 func pushBundle(dockerCli command.Cli, opts pushOptions, bndl *bundle.Bundle, tag reference.Named) error {
@@ -179,9 +223,9 @@ func pushBundle(dockerCli command.Cli, opts pushOptions, bndl *bundle.Bundle, ta
 	if platforms := platformFilter(opts); len(platforms) > 0 {
 		fixupOptions = append(fixupOptions, remotes.WithComponentImagePlatforms(platforms))
 	}
-	// bundle fixup
-	if err := remotes.FixupBundle(context.Background(), bndl, tag, resolver, fixupOptions...); err != nil {
-		return errors.Wrapf(err, "fixing up %q for push", tag)
+	if logrus.IsLevelEnabled(logrus.DebugLevel) {
+		bt, _ := json.MarshalIndent(bndl, "> ", "  ")
+		fmt.Println(string(bt))
 	}
 	// push bundle manifest
 	logrus.Debugf("Pushing the bundle %q", tag)