enable multi export

api/services/control/control.proto

@@ -53,17 +53,23 @@ message UsageRecord {
 	bool Shared = 11;
 }
 
+message ExporterAttrs {
+    map<string, string> exporterAttrs = 1;
+}
+
 message SolveRequest {
 	string Ref = 1;
 	pb.Definition Definition = 2;
 	string Exporter = 3;
-	map<string, string> ExporterAttrs = 4;
-	string Session = 5;
-	string Frontend = 6;
-	map<string, string> FrontendAttrs = 7;
-	CacheOptions Cache = 8 [(gogoproto.nullable) = false];
-	repeated string Entitlements = 9 [(gogoproto.customtype) = "github.com/moby/buildkit/util/entitlements.Entitlement" ];
-	map<string, pb.Definition> FrontendInputs = 10;
+	ExporterAttrs ExporterAttrs = 4;
+	repeated string Exporters = 5;
+	repeated ExporterAttrs ExportersAttrs = 6;
+	string Session = 7;
+	string Frontend = 8;
+	map<string, string> FrontendAttrs = 9;
+	CacheOptions Cache = 10 [(gogoproto.nullable) = false];
+	repeated string Entitlements = 11 [(gogoproto.customtype) = "github.com/moby/buildkit/util/entitlements.Entitlement" ];
+	map<string, pb.Definition> FrontendInputs = 12;
 }
 
 message CacheOptions {
@@ -94,9 +100,13 @@ message CacheOptionsEntry {
 	map<string, string> Attrs = 2;
 }
 
-message SolveResponse {
+message ExporterResponse{
 	map<string, string> ExporterResponse = 1;
 }
+message SolveResponse {
+	ExporterResponse ExporterResponse = 1;
+	repeated ExporterResponse ExportersResponse= 2;
+}
 
 message StatusRequest {
 	string Ref = 1;

client/client_test.go

@@ -582,12 +582,15 @@ func testPushByDigest(t *testing.T, sb integration.Sandbox) {
 	_, _, err = contentutil.ProviderFromRef(name + ":latest")
 	require.Error(t, err)
 
-	desc, _, err := contentutil.ProviderFromRef(name + "@" + resp.ExporterResponse["containerimage.digest"])
-	require.NoError(t, err)
-
-	require.Equal(t, resp.ExporterResponse["containerimage.digest"], desc.Digest.String())
-	require.Equal(t, images.MediaTypeDockerSchema2Manifest, desc.MediaType)
-	require.True(t, desc.Size > 0)
+	if len(resp.ExportersResponse) > 0 {
+		for _, e := range resp.ExportersResponse {
+			desc, _, err := contentutil.ProviderFromRef(name + "@" + e.ExporterResponse["containerimage.digest"])
+			require.NoError(t, err)
+			require.Equal(t, e.ExporterResponse["containerimage.digest"], desc.Digest.String())
+			require.Equal(t, images.MediaTypeDockerSchema2Manifest, desc.MediaType)
+			require.True(t, desc.Size > 0)
+		}
+	}
 }
 
 func testSecurityMode(t *testing.T, sb integration.Sandbox) {
@@ -850,8 +853,11 @@ func testFrontendImageNaming(t *testing.T, sb integration.Sandbox) {
 
 					resp, err := c.Build(context.TODO(), so, "", frontend, nil)
 					require.NoError(t, err)
-
-					checkImageName[exp](out, imageName, resp.ExporterResponse)
+					if len(resp.ExportersResponse) > 0 {
+						for _, e := range resp.ExportersResponse {
+							checkImageName[exp](out, imageName, e.ExporterResponse)
+						}
+					}
 				})
 			}
 		})
@@ -1539,10 +1545,14 @@ func testFrontendMetadataReturn(t *testing.T, sb integration.Sandbox) {
 		},
 	}, "", frontend, nil)
 	require.NoError(t, err)
-	require.Contains(t, res.ExporterResponse, "frontend.returned")
-	require.Equal(t, res.ExporterResponse["frontend.returned"], "true")
-	require.NotContains(t, res.ExporterResponse, "not-frontend.not-returned")
-	require.NotContains(t, res.ExporterResponse, "frontendnot.returned.either")
+	if len(res.ExportersResponse) > 0 {
+		for _, e := range res.ExportersResponse {
+			require.Contains(t, e.ExporterResponse, "frontend.returned")
+			require.Equal(t, e.ExporterResponse["frontend.returned"], "true")
+			require.NotContains(t, e.ExporterResponse, "not-frontend.not-returned")
+			require.NotContains(t, e.ExporterResponse, "frontendnot.returned.either")
+		}
+	}
 	checkAllReleasable(t, c, sb, true)
 }
 
@@ -1645,10 +1655,14 @@ func testExporterTargetExists(t *testing.T, sb integration.Sandbox) {
 		},
 	}, nil)
 	require.NoError(t, err)
-	dgst := res.ExporterResponse["containerimage.digest"]
+	if len(res.ExportersResponse) > 0 {
+		for _, e := range res.ExportersResponse {
+			dgst := e.ExporterResponse["containerimage.digest"]
 
-	require.True(t, strings.HasPrefix(dgst, "sha256:"))
-	require.Equal(t, dgst, mdDgst)
+			require.True(t, strings.HasPrefix(dgst, "sha256:"))
+			require.Equal(t, dgst, mdDgst)
+		}
+	}
 }
 
 func testTarExporterWithSocket(t *testing.T, sb integration.Sandbox) {
@@ -2491,18 +2505,12 @@ func testBasicInlineCacheImportExport(t *testing.T, sb integration.Sandbox) {
 	}, nil)
 	require.NoError(t, err)
 
-	dgst, ok := resp.ExporterResponse["containerimage.digest"]
-	require.Equal(t, ok, true)
-
-	unique, err := readFileInImage(c, target+"@"+dgst, "/unique")
-	require.NoError(t, err)
-
 	err = c.Prune(context.TODO(), nil, PruneAll)
 	require.NoError(t, err)
 
 	checkAllRemoved(t, c, sb)
 
-	resp, err = c.Solve(context.TODO(), def, SolveOpt{
+	res, err := c.Solve(context.TODO(), def, SolveOpt{
 		// specifying inline cache exporter is needed for reproducing containerimage.digest
 		// (not needed for reproducing rootfs/unique)
 		Exports: []ExportEntry{
@@ -2530,17 +2538,26 @@ func testBasicInlineCacheImportExport(t *testing.T, sb integration.Sandbox) {
 	}, nil)
 	require.NoError(t, err)
 
-	dgst2, ok := resp.ExporterResponse["containerimage.digest"]
-	require.Equal(t, ok, true)
+	if len(resp.ExportersResponse) > 0 && len(res.ExportersResponse) > 0 {
+		for _, e := range res.ExportersResponse {
+			for _, v := range resp.ExportersResponse {
+				dgst, ok := v.ExporterResponse["containerimage.digest"]
+				require.Equal(t, ok, true)
 
-	require.Equal(t, dgst, dgst2)
+				dgst2, ok := e.ExporterResponse["containerimage.digest"]
+				require.Equal(t, ok, true)
+
+				require.Equal(t, dgst, dgst2)
+			}
+		}
+	}
 
 	err = c.Prune(context.TODO(), nil, PruneAll)
 	require.NoError(t, err)
 
 	checkAllRemoved(t, c, sb)
 
-	resp, err = c.Solve(context.TODO(), def, SolveOpt{
+	res, err = c.Solve(context.TODO(), def, SolveOpt{
 		Exports: []ExportEntry{
 			{
 				Type: ExporterImage,
@@ -2561,13 +2578,27 @@ func testBasicInlineCacheImportExport(t *testing.T, sb integration.Sandbox) {
 	}, nil)
 	require.NoError(t, err)
 
-	dgst3, ok := resp.ExporterResponse["containerimage.digest"]
-	require.Equal(t, ok, true)
+	if len(resp.ExportersResponse) > 0 && len(res.ExportersResponse) > 0 {
+		for _, e := range resp.ExportersResponse {
+			for _, v := range res.ExportersResponse {
+				dgst3, ok := v.ExporterResponse["containerimage.digest"]
+				require.Equal(t, ok, true)
+
+				dgst, ok := e.ExporterResponse["containerimage.digest"]
+				require.Equal(t, ok, true)
+
+				unique, err := readFileInImage(c, target+"@"+dgst, "/unique")
+				require.NoError(t, err)
+
+				// dgst3 != dgst, because inline cache is not exported for dgst3
+				unique3, err := readFileInImage(c, target+"@"+dgst3, "/unique")
+				require.NoError(t, err)
+
+				require.EqualValues(t, unique, unique3)
+			}
+		}
+	}
 
-	// dgst3 != dgst, because inline cache is not exported for dgst3
-	unique3, err := readFileInImage(c, target+"@"+dgst3, "/unique")
-	require.NoError(t, err)
-	require.EqualValues(t, unique, unique3)
 }
 
 func readFileInImage(c *Client, ref, path string) ([]byte, error) {

client/graph.go

@@ -3,6 +3,7 @@ package client
 import (
 	"time"
 
+	controlapi "github.com/moby/buildkit/api/services/control"
 	digest "github.com/opencontainers/go-digest"
 )
 
@@ -42,5 +43,5 @@ type SolveStatus struct {
 
 type SolveResponse struct {
 	// ExporterResponse is also used for CacheExporter
-	ExporterResponse map[string]string
+	ExportersResponse []*controlapi.ExporterResponse
 }

client/solve.go

@@ -114,14 +114,6 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 		return nil, err
 	}
 
-	var ex ExportEntry
-	if len(opt.Exports) > 1 {
-		return nil, errors.New("currently only single Exports can be specified")
-	}
-	if len(opt.Exports) == 1 {
-		ex = opt.Exports[0]
-	}
-
 	if !opt.SessionPreInitialized {
 		if len(syncedDirs) > 0 {
 			s.Allow(filesync.NewFSSyncProvider(syncedDirs))
@@ -131,29 +123,50 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 			s.Allow(a)
 		}
 
-		switch ex.Type {
-		case ExporterLocal:
-			if ex.Output != nil {
-				return nil, errors.New("output file writer is not supported by local exporter")
-			}
-			if ex.OutputDir == "" {
-				return nil, errors.New("output directory is required for local exporter")
-			}
-			s.Allow(filesync.NewFSSyncTargetDir(ex.OutputDir))
-		case ExporterOCI, ExporterDocker, ExporterTar:
-			if ex.OutputDir != "" {
-				return nil, errors.Errorf("output directory %s is not supported by %s exporter", ex.OutputDir, ex.Type)
-			}
-			if ex.Output == nil {
-				return nil, errors.Errorf("output file writer is required for %s exporter", ex.Type)
-			}
-			s.Allow(filesync.NewFSSyncTarget(ex.Output))
-		default:
-			if ex.Output != nil {
-				return nil, errors.Errorf("output file writer is not supported by %s exporter", ex.Type)
-			}
-			if ex.OutputDir != "" {
-				return nil, errors.Errorf("output directory %s is not supported by %s exporter", ex.OutputDir, ex.Type)
+		duplicateLocalExporter := 0
+		duplicateOciExporter := 0
+		duplicateDockerExporter := 0
+		duplicateTarExporter := 0
+		for _, ex := range opt.Exports {
+			switch ex.Type {
+			case ExporterLocal:
+				duplicateLocalExporter++
+				if duplicateLocalExporter > 1 {
+					return nil, errors.New("using multiple ExporterLocal is not supported")
+				}
+				if ex.Output != nil {
+					return nil, errors.New("output file writer is not supported by local exporter")
+				}
+				if ex.OutputDir == "" {
+					return nil, errors.New("output directory is required for local exporter")
+				}
+				s.Allow(filesync.NewFSSyncTargetDir(ex.OutputDir))
+			case ExporterOCI, ExporterDocker, ExporterTar:
+				switch ex.Type {
+				case ExporterOCI:
+					duplicateOciExporter++
+				case ExporterDocker:
+					duplicateDockerExporter++
+				case ExporterTar:
+					duplicateTarExporter++
+				}
+				if duplicateOciExporter > 1 || duplicateDockerExporter > 1 || duplicateTarExporter > 1 {
+					return nil, errors.New("using multiple ExporterOCI is not supported")
+				}
+				if ex.OutputDir != "" {
+					return nil, errors.Errorf("output directory %s is not supported by %s exporter", ex.OutputDir, ex.Type)
+				}
+				if ex.Output == nil {
+					return nil, errors.Errorf("output file writer is required for %s exporter", ex.Type)
+				}
+				s.Allow(filesync.NewFSSyncTarget(ex.Output))
+			default:
+				if ex.Output != nil {
+					return nil, errors.Errorf("output file writer is not supported by %s exporter", ex.Type)
+				}
+				if ex.OutputDir != "" {
+					return nil, errors.Errorf("output directory %s is not supported by %s exporter", ex.OutputDir, ex.Type)
+				}
 			}
 		}
 
@@ -198,11 +211,40 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 			frontendInputs[key] = def.ToPB()
 		}
 
+		var exportersTypes []string
+		var exporterType string
+		m := &controlapi.SolveRequest{
+			ExportersAttrs: []*controlapi.ExporterAttrs{},
+			ExporterAttrs:  &controlapi.ExporterAttrs{},
+		}
+		expo := &controlapi.ExporterAttrs{
+			ExporterAttrs: make(map[string]string),
+		}
+
+		if len(opt.Exports) > 1 {
+			exporterType = ""
+			m.ExporterAttrs = nil
+			for _, ex := range opt.Exports {
+				exportersTypes = append(exportersTypes, ex.Type)
+				expo.ExporterAttrs = ex.Attrs
+				m.ExportersAttrs = append(m.ExportersAttrs, expo)
+			}
+		}
+		if len(opt.Exports) == 1 {
+			exportersTypes = nil
+			m.ExportersAttrs = nil
+			exporterType = opt.Exports[0].Type
+			expo.ExporterAttrs = opt.Exports[0].Attrs
+			m.ExporterAttrs = expo
+		}
+
 		resp, err := c.controlClient().Solve(ctx, &controlapi.SolveRequest{
 			Ref:            ref,
 			Definition:     pbd,
-			Exporter:       ex.Type,
-			ExporterAttrs:  ex.Attrs,
+			Exporters:      exportersTypes,
+			ExportersAttrs: m.ExportersAttrs,
+			Exporter:       exporterType,
+			ExporterAttrs:  m.ExporterAttrs,
 			Session:        s.ID(),
 			Frontend:       opt.Frontend,
 			FrontendAttrs:  opt.FrontendAttrs,
@@ -213,8 +255,13 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 		if err != nil {
 			return errors.Wrap(err, "failed to solve")
 		}
+
+		var exportersResponse []*controlapi.ExporterResponse
+		for _, v := range resp.ExportersResponse {
+			exportersResponse = append(exportersResponse, v)
+		}
 		res = &SolveResponse{
-			ExporterResponse: resp.ExporterResponse,
+			ExportersResponse: exportersResponse,
 		}
 		return nil
 	})
@@ -299,14 +346,18 @@ func (c *Client) solve(ctx context.Context, def *llb.Definition, runGateway runG
 	}
 	// Update index.json of exported cache content store
 	// FIXME(AkihiroSuda): dedupe const definition of cache/remotecache.ExporterResponseManifestDesc = "cache.manifest"
-	if manifestDescJSON := res.ExporterResponse["cache.manifest"]; manifestDescJSON != "" {
-		var manifestDesc ocispec.Descriptor
-		if err = json.Unmarshal([]byte(manifestDescJSON), &manifestDesc); err != nil {
-			return nil, err
-		}
-		for indexJSONPath, tag := range cacheOpt.indicesToUpdate {
-			if err = ociindex.PutDescToIndexJSONFileLocked(indexJSONPath, manifestDesc, tag); err != nil {
-				return nil, err
+	if len(res.ExportersResponse) > 0 {
+		for _, v := range res.ExportersResponse {
+			if manifestDescJSON := v.ExporterResponse["cache.manifest"]; manifestDescJSON != "" {
+				var manifestDesc ocispec.Descriptor
+				if err = json.Unmarshal([]byte(manifestDescJSON), &manifestDesc); err != nil {
+					return nil, err
+				}
+				for indexJSONPath, tag := range cacheOpt.indicesToUpdate {
+					if err = ociindex.PutDescToIndexJSONFileLocked(indexJSONPath, manifestDesc, tag); err != nil {
+						return nil, err
+					}
+				}
 			}
 		}
 	}