feat: allow ignoring remote cache-export error if failing

Signed-off-by: JordanGoasdoue <jordan.goasdoue@dailymotion.com>

README.md

@@ -390,6 +390,7 @@ buildctl build ... \
 * `compression=<uncompressed|gzip|estargz|zstd>`: choose compression type for layers newly created and cached, gzip is default value. estargz and zstd should be used with `oci-mediatypes=true`
 * `compression-level=<value>`: choose compression level for gzip, estargz (0-9) and zstd (0-22)
 * `force-compression=true`: forcibly apply `compression` option to all layers
+* `ignore-error=<false|true>`: specify if error is ignored in case cache export fails (default: `false`)
 
 `--import-cache` options:
 * `type=registry`
@@ -415,6 +416,7 @@ The directory layout conforms to OCI Image Spec v1.0.
 * `compression=<uncompressed|gzip|estargz|zstd>`: choose compression type for layers newly created and cached, gzip is default value. estargz and zstd should be used with `oci-mediatypes=true`.
 * `compression-level=<value>`: compression level for gzip, estargz (0-9) and zstd (0-22)
 * `force-compression=true`: forcibly apply `compression` option to all layers
+* `ignore-error=<false|true>`: specify if error is ignored in case cache export fails (default: `false`)
 
 `--import-cache` options:
 * `type=local`
@@ -449,6 +451,7 @@ in your workflow to expose the runtime.
   * `min`: only export layers for the resulting image
   * `max`: export all the layers of all intermediate steps
 * `scope=<scope>`: which scope cache object belongs to (default `buildkit`)
+* `ignore-error=<false|true>`: specify if error is ignored in case cache export fails (default: `false`)
 
 `--import-cache` options:
 * `type=gha`
@@ -496,6 +499,7 @@ Others options are:
 * `prefix=<prefix>`: set global prefix to store / read files on s3 (default: empty)
 * `name=<manifest>`: specify name of the manifest to use (default `buildkit`)
   * Multiple manifest names can be specified at the same time, separated by `;`. The standard use case is to use the git sha1 as name, and the branch name as duplicate, and load both with 2 `import-cache` commands.
+* `ignore-error=<false|true>`: specify if error is ignored in case cache export fails (default: `false`)
 
 `--import-cache` options:
 * `type=s3`
@@ -540,6 +544,7 @@ There are 2 options supported for Azure Blob Storage authentication:
 * `prefix=<prefix>`: set global prefix to store / read files on the Azure Blob Storage container (`<container>`) (default: empty)
 * `name=<manifest>`: specify name of the manifest to use (default: `buildkit`)
   * Multiple manifest names can be specified at the same time, separated by `;`. The standard use case is to use the git sha1 as name, and the branch name as duplicate, and load both with 2 `import-cache` commands.
+* `ignore-error=<false|true>`: specify if error is ignored in case cache export fails (default: `false`)
 
 `--import-cache` options:
 * `type=azblob`

control/control.go

@@ -3,6 +3,7 @@ package control
 import (
 	"context"
 	"fmt"
+	"strconv"
 	"sync"
 	"sync/atomic"
 	"time"
@@ -368,6 +369,14 @@ func (c *Controller) Solve(ctx context.Context, req *controlapi.SolveRequest) (*
 		} else {
 			exp.CacheExportMode = exportMode
 		}
+		if ignoreErrorStr, ok := e.Attrs["ignore-error"]; ok {
+			ignoreError, err := strconv.ParseBool(ignoreErrorStr)
+			if err != nil {
+				bklog.G(ctx).Debugf("skipping invalid cache export ignore-error: %s", e.Attrs["ignore-error"])
+			} else {
+				exp.IgnoreError = ignoreError
+			}
+		}
 		cacheExporters = append(cacheExporters, exp)
 	}
 

hack/s3_test/test.sh

@@ -20,7 +20,17 @@ done
 
 export default_options="type=s3,bucket=my-bucket,region=us-east-1,endpoint_url=http://127.0.0.1:9000,access_key_id=minioadmin,secret_access_key=minioadmin,use_path_style=true"
 
-rm -rf /tmp/destdir1 /tmp/destdir2
+rm -rf /tmp/destdir0 /tmp/destdir1 /tmp/destdir2
+
+# Check --export-cache with ignore-error=true and fake non reachable endpoint_url
+# Should ignore error and skip the remote cache export
+buildctl build \
+  --progress plain \
+  --frontend dockerfile.v0 \
+  --local context=/test/test1 \
+  --local dockerfile=/test/test1 \
+  --export-cache "$default_options,endpoint_url=http://fake-url:9000,ignore-error=true" \
+  --output type=local,dest=/tmp/destdir0
 
 # First build: no cache on s3
 # 4 files should be exported (2 blobs + 2 manifests)

solver/llbsolver/solver.go

@@ -58,6 +58,7 @@ type ExporterRequest struct {
 type RemoteCacheExporter struct {
 	remotecache.Exporter
 	solver.CacheExportMode
+	IgnoreError bool
 }
 
 // ResolveWorkerFunc returns default worker for the temporary default non-distributed use cases
@@ -570,7 +571,7 @@ func runCacheExporters(ctx context.Context, exporters []RemoteCacheExporter, j *
 		func(exp RemoteCacheExporter, i int) {
 			eg.Go(func() (err error) {
 				id := fmt.Sprint(j.SessionID, "-cache-", i)
-				return inBuilderContext(ctx, j, exp.Exporter.Name(), id, func(ctx context.Context, _ session.Group) error {
+				err = inBuilderContext(ctx, j, exp.Exporter.Name(), id, func(ctx context.Context, _ session.Group) error {
 					prepareDone := progress.OneOff(ctx, "preparing build cache for export")
 					if err := result.EachRef(cached, inp, func(res solver.CachedResult, ref cache.ImmutableRef) error {
 						ctx = withDescHandlerCacheOpts(ctx, ref)
@@ -589,13 +590,13 @@ func runCacheExporters(ctx context.Context, exporters []RemoteCacheExporter, j *
 					}); err != nil {
 						return prepareDone(err)
 					}
-					prepareDone(nil)
 					resps[i], err = exp.Finalize(ctx)
-					if err != nil {
-						return err
-					}
-					return nil
+					return prepareDone(err)
 				})
+				if exp.IgnoreError {
+					err = nil
+				}
+				return err
 			})
 		}(exp, i)
 	}