llbsolver: move history blobs to a separate namespace

Migrate history objects to separate namespace to holding
reference to a blob does not interfer with the GC labels
held for same blobs by the containerd image store.

Signed-off-by: Tonis Tiigi <tonistiigi@gmail.com>

control/control.go

@@ -10,7 +10,6 @@ import (
 
 	contentapi "github.com/containerd/containerd/api/services/content/v1"
 	"github.com/containerd/containerd/content"
-	"github.com/containerd/containerd/leases"
 	"github.com/containerd/containerd/services/content/contentserver"
 	"github.com/docker/distribution/reference"
 	"github.com/mitchellh/hashstructure/v2"
@@ -26,12 +25,14 @@ import (
 	"github.com/moby/buildkit/frontend/attestations"
 	"github.com/moby/buildkit/session"
 	"github.com/moby/buildkit/session/grpchijack"
+	containerdsnapshot "github.com/moby/buildkit/snapshot/containerd"
 	"github.com/moby/buildkit/solver"
 	"github.com/moby/buildkit/solver/llbsolver"
 	"github.com/moby/buildkit/solver/llbsolver/proc"
 	"github.com/moby/buildkit/solver/pb"
 	"github.com/moby/buildkit/util/bklog"
 	"github.com/moby/buildkit/util/imageutil"
+	"github.com/moby/buildkit/util/leaseutil"
 	"github.com/moby/buildkit/util/throttle"
 	"github.com/moby/buildkit/util/tracing/transform"
 	"github.com/moby/buildkit/version"
@@ -58,8 +59,8 @@ type Opt struct {
 	Entitlements              []string
 	TraceCollector            sdktrace.SpanExporter
 	HistoryDB                 *bbolt.DB
-	LeaseManager              leases.Manager
-	ContentStore              content.Store
+	LeaseManager              *leaseutil.Manager
+	ContentStore              *containerdsnapshot.Store
 	HistoryConfig             *config.HistoryConfig
 }
 
@@ -79,12 +80,15 @@ type Controller struct { // TODO: ControlService
 func NewController(opt Opt) (*Controller, error) {
 	gatewayForwarder := controlgateway.NewGatewayForwarder()
 
-	hq := llbsolver.NewHistoryQueue(llbsolver.HistoryQueueOpt{
+	hq, err := llbsolver.NewHistoryQueue(llbsolver.HistoryQueueOpt{
 		DB:           opt.HistoryDB,
 		LeaseManager: opt.LeaseManager,
 		ContentStore: opt.ContentStore,
 		CleanConfig:  opt.HistoryConfig,
 	})
+	if err != nil {
+		return nil, errors.Wrap(err, "failed to create history queue")
+	}
 
 	s, err := llbsolver.New(llbsolver.Opt{
 		WorkerController: opt.WorkerController,
@@ -125,7 +129,7 @@ func (c *Controller) Register(server *grpc.Server) {
 	c.gatewayForwarder.Register(server)
 	tracev1.RegisterTraceServiceServer(server, c)
 
-	store := &roContentStore{c.opt.ContentStore}
+	store := &roContentStore{c.opt.ContentStore.WithFallbackNS(c.opt.ContentStore.Namespace() + "_history")}
 	contentapi.RegisterContentServer(server, contentserver.New(store))
 }
 

snapshot/containerd/content.go

@@ -5,12 +5,15 @@ import (
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/namespaces"
+	"github.com/containerd/nydus-snapshotter/pkg/errdefs"
 	digest "github.com/opencontainers/go-digest"
 	ocispecs "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/pkg/errors"
 )
 
-func NewContentStore(store content.Store, ns string) content.Store {
+type Store = nsContent
+
+func NewContentStore(store content.Store, ns string) *Store {
 	return &nsContent{ns, store}
 }
 
@@ -19,6 +22,14 @@ type nsContent struct {
 	content.Store
 }
 
+func (c *nsContent) Namespace() string {
+	return c.ns
+}
+
+func (c *nsContent) WithNamespace(ns string) *Store {
+	return NewContentStore(c.Store, ns)
+}
+
 func (c *nsContent) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
 	ctx = namespaces.WithNamespace(ctx, c.ns)
 	return c.Store.Info(ctx, dgst)
@@ -62,6 +73,13 @@ func (c *nsContent) Writer(ctx context.Context, opts ...content.WriterOpt) (cont
 	return c.writer(ctx, 3, opts...)
 }
 
+func (c *nsContent) WithFallbackNS(ns string) content.Store {
+	return &nsFallbackStore{
+		main: c,
+		fb:   c.WithNamespace(ns),
+	}
+}
+
 func (c *nsContent) writer(ctx context.Context, retries int, opts ...content.WriterOpt) (content.Writer, error) {
 	ctx = namespaces.WithNamespace(ctx, c.ns)
 	w, err := c.Store.Writer(ctx, opts...)
@@ -80,3 +98,58 @@ func (w *nsWriter) Commit(ctx context.Context, size int64, expected digest.Diges
 	ctx = namespaces.WithNamespace(ctx, w.ns)
 	return w.Writer.Commit(ctx, size, expected, opts...)
 }
+
+type nsFallbackStore struct {
+	main *nsContent
+	fb   *nsContent
+}
+
+var _ content.Store = &nsFallbackStore{}
+
+func (c *nsFallbackStore) Info(ctx context.Context, dgst digest.Digest) (content.Info, error) {
+	info, err := c.main.Info(ctx, dgst)
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			return c.fb.Info(ctx, dgst)
+		}
+	}
+	return info, err
+}
+
+func (c *nsFallbackStore) Update(ctx context.Context, info content.Info, fieldpaths ...string) (content.Info, error) {
+	return c.main.Update(ctx, info, fieldpaths...)
+}
+
+func (c *nsFallbackStore) Walk(ctx context.Context, fn content.WalkFunc, filters ...string) error {
+	return c.main.Walk(ctx, fn, filters...)
+}
+
+func (c *nsFallbackStore) Delete(ctx context.Context, dgst digest.Digest) error {
+	return c.main.Delete(ctx, dgst)
+}
+
+func (c *nsFallbackStore) Status(ctx context.Context, ref string) (content.Status, error) {
+	return c.main.Status(ctx, ref)
+}
+
+func (c *nsFallbackStore) ListStatuses(ctx context.Context, filters ...string) ([]content.Status, error) {
+	return c.main.ListStatuses(ctx, filters...)
+}
+
+func (c *nsFallbackStore) Abort(ctx context.Context, ref string) error {
+	return c.main.Abort(ctx, ref)
+}
+
+func (c *nsFallbackStore) ReaderAt(ctx context.Context, desc ocispecs.Descriptor) (content.ReaderAt, error) {
+	ra, err := c.main.ReaderAt(ctx, desc)
+	if err != nil {
+		if errdefs.IsNotFound(err) {
+			return c.fb.ReaderAt(ctx, desc)
+		}
+	}
+	return ra, err
+}
+
+func (c *nsFallbackStore) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
+	return c.main.Writer(ctx, opts...)
+}