bump up containerd (v1.0.0-beta.3)

Signed-off-by: Akihiro Suda <suda.akihiro@lab.ntt.co.jp>

README.md

@@ -123,7 +123,7 @@ The `tonistiigi/buildkit:standalone` image can be built locally using the Docker
 
 #### Supported runc version
 
-During development buildkit is tested with the version of runc that is being used by the containerd repository. Please refer to [runc.md](https://github.com/containerd/containerd/blob/d1e11f17ec7b325f89608dd46c128300b8727d50/RUNC.md) for more information.
+During development buildkit is tested with the version of runc that is being used by the containerd repository. Please refer to [runc.md](https://github.com/containerd/containerd/blob/v1.0.0-beta.3/RUNC.md) for more information.
 
 
 #### Contributing

cache/blobs/blobs.go

@@ -3,8 +3,8 @@ package blobs
 import (
 	gocontext "context"
 
+	"github.com/containerd/containerd/diff"
 	"github.com/containerd/containerd/mount"
-	"github.com/containerd/containerd/rootfs"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/snapshot"
 	"github.com/moby/buildkit/util/flightcontrol"
@@ -27,7 +27,7 @@ type blobmapper interface {
 	SetBlob(ctx gocontext.Context, key string, diffID, blob digest.Digest) error
 }
 
-func GetDiffPairs(ctx context.Context, snapshotter snapshot.Snapshotter, differ rootfs.MountDiffer, ref cache.ImmutableRef) ([]DiffPair, error) {
+func GetDiffPairs(ctx context.Context, snapshotter snapshot.Snapshotter, differ diff.Differ, ref cache.ImmutableRef) ([]DiffPair, error) {
 	blobmap, ok := snapshotter.(blobmapper)
 	if !ok {
 		return nil, errors.Errorf("image exporter requires snapshotter with blobs mapping support")
@@ -71,7 +71,9 @@ func GetDiffPairs(ctx context.Context, snapshotter snapshot.Snapshotter, differ
 			if err != nil {
 				return nil, err
 			}
-			descr, err := differ.DiffMounts(ctx, lower, upper, ocispec.MediaTypeImageLayer, ref.ID())
+			descr, err := differ.DiffMounts(ctx, lower, upper,
+				diff.WithMediaType(ocispec.MediaTypeImageLayer),
+				diff.WithReference(ref.ID()))
 			if err != nil {
 				return nil, err
 			}

cache/cacheimport/export.go

@@ -6,8 +6,8 @@ import (
 	"encoding/json"
 
 	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/diff"
 	"github.com/containerd/containerd/images"
-	"github.com/containerd/containerd/rootfs"
 	"github.com/docker/distribution/manifest"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/moby/buildkit/cache"
@@ -38,7 +38,7 @@ type CacheRecord struct {
 type ExporterOpt struct {
 	Snapshotter    snapshot.Snapshotter
 	ContentStore   content.Store
-	Differ         rootfs.MountDiffer
+	Differ         diff.Differ
 	SessionManager *session.Manager
 }
 

cache/cacheimport/import.go

@@ -7,9 +7,11 @@ import (
 	"time"
 
 	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/diff"
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/containerd/remotes/docker"
 	"github.com/containerd/containerd/rootfs"
+	cdsnapshot "github.com/containerd/containerd/snapshot"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/cache/blobs"
 	"github.com/moby/buildkit/client"
@@ -30,7 +32,7 @@ type ImportOpt struct {
 	SessionManager *session.Manager
 	ContentStore   content.Store
 	Snapshotter    snapshot.Snapshotter
-	Applier        rootfs.Applier
+	Applier        diff.Differ
 	CacheAccessor  cache.Accessor
 }
 
@@ -263,10 +265,18 @@ func (ii *importInfo) unpack(ctx context.Context, dpairs []blobs.DiffPair) (stri
 		return "", err
 	}
 
-	chainID, err := rootfs.ApplyLayers(ctx, layers, ii.opt.Snapshotter, ii.opt.Applier)
-	if err != nil {
-		return "", err
+	var chain []digest.Digest
+	for _, layer := range layers {
+		labels := map[string]string{
+			"containerd.io/gc.root":      time.Now().UTC().Format(time.RFC3339Nano),
+			"containerd.io/uncompressed": layer.Diff.Digest.String(),
+		}
+		if _, err := rootfs.ApplyLayer(ctx, layer, chain, ii.opt.Snapshotter, ii.opt.Applier, cdsnapshot.WithLabels(labels)); err != nil {
+			return "", err
+		}
+		chain = append(chain, layer.Diff.Digest)
 	}
+	chainID := identity.ChainID(chain)
 
 	if err := ii.fillBlobMapping(ctx, layers); err != nil {
 		return "", err

cache/contenthash/checksum_test.go

@@ -33,7 +33,9 @@ func TestChecksumBasicFile(t *testing.T) {
 	require.NoError(t, err)
 	defer os.RemoveAll(tmpdir)
 
-	cm := setupCacheManager(t, tmpdir)
+	snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
+	require.NoError(t, err)
+	cm := setupCacheManager(t, tmpdir, snapshotter)
 	defer cm.Close()
 
 	ch := []string{
@@ -180,7 +182,9 @@ func TestHandleChange(t *testing.T) {
 	require.NoError(t, err)
 	defer os.RemoveAll(tmpdir)
 
-	cm := setupCacheManager(t, tmpdir)
+	snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
+	require.NoError(t, err)
+	cm := setupCacheManager(t, tmpdir, snapshotter)
 	defer cm.Close()
 
 	ch := []string{
@@ -255,7 +259,9 @@ func TestPersistence(t *testing.T) {
 	require.NoError(t, err)
 	defer os.RemoveAll(tmpdir)
 
-	cm := setupCacheManager(t, tmpdir)
+	snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
+	require.NoError(t, err)
+	cm := setupCacheManager(t, tmpdir, snapshotter)
 	defer cm.Close()
 
 	ch := []string{
@@ -289,9 +295,10 @@ func TestPersistence(t *testing.T) {
 
 	time.Sleep(100 * time.Millisecond) // saving happens on the background
 
+	// we can't close snapshotter and open it twice (especially, its internal boltdb store)
 	cm.Close()
 	getDefaultManager().lru.Purge()
-	cm = setupCacheManager(t, tmpdir)
+	cm = setupCacheManager(t, tmpdir, snapshotter)
 	defer cm.Close()
 
 	ref, err = cm.Get(context.TODO(), id)
@@ -324,10 +331,7 @@ func createRef(t *testing.T, cm cache.Manager, files []string) cache.ImmutableRe
 	return ref
 }
 
-func setupCacheManager(t *testing.T, tmpdir string) cache.Manager {
-	snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
-	require.NoError(t, err)
-
+func setupCacheManager(t *testing.T, tmpdir string, snapshotter snapshot.Snapshotter) cache.Manager {
 	md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db"))
 	require.NoError(t, err)
 

cache/manager.go

@@ -188,7 +188,10 @@ func (cm *cacheManager) New(ctx context.Context, s ImmutableRef, opts ...RefOpti
 		parentID = parent.ID()
 	}
 
-	if _, err := cm.Snapshotter.Prepare(ctx, id, parentID); err != nil {
+	labels := map[string]string{
+		"containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339Nano),
+	}
+	if _, err := cm.Snapshotter.Prepare(ctx, id, parentID, cdsnapshot.WithLabels(labels)); err != nil {
 		if parent != nil {
 			parent.Release(context.TODO())
 		}

cache/manager_test.go

@@ -24,7 +24,9 @@ func TestManager(t *testing.T) {
 	require.NoError(t, err)
 	defer os.RemoveAll(tmpdir)
 
-	cm := getCacheManager(t, tmpdir)
+	snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
+	require.NoError(t, err)
+	cm := getCacheManager(t, tmpdir, snapshotter)
 
 	_, err = cm.Get(ctx, "foobar")
 	require.Error(t, err)
@@ -132,7 +134,9 @@ func TestLazyCommit(t *testing.T) {
 	require.NoError(t, err)
 	defer os.RemoveAll(tmpdir)
 
-	cm := getCacheManager(t, tmpdir)
+	snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
+	require.NoError(t, err)
+	cm := getCacheManager(t, tmpdir, snapshotter)
 
 	active, err := cm.New(ctx, nil, CachePolicyRetain)
 	require.NoError(t, err)
@@ -213,7 +217,8 @@ func TestLazyCommit(t *testing.T) {
 	err = cm.Close()
 	require.NoError(t, err)
 
-	cm = getCacheManager(t, tmpdir)
+	// we can't close snapshotter and open it twice (especially, its internal boltdb store)
+	cm = getCacheManager(t, tmpdir, snapshotter)
 
 	snap2, err = cm.Get(ctx, snap.ID())
 	require.NoError(t, err)
@@ -234,7 +239,7 @@ func TestLazyCommit(t *testing.T) {
 	err = cm.Close()
 	require.NoError(t, err)
 
-	cm = getCacheManager(t, tmpdir)
+	cm = getCacheManager(t, tmpdir, snapshotter)
 
 	snap2, err = cm.Get(ctx, snap.ID())
 	require.NoError(t, err)
@@ -250,10 +255,7 @@ func TestLazyCommit(t *testing.T) {
 	require.Equal(t, errNotFound, errors.Cause(err))
 }
 
-func getCacheManager(t *testing.T, tmpdir string) Manager {
-	snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
-	require.NoError(t, err)
-
+func getCacheManager(t *testing.T, tmpdir string, snapshotter snapshot.Snapshotter) Manager {
 	md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db"))
 	require.NoError(t, err)
 

cache/refs.go

@@ -2,8 +2,10 @@ package cache
 
 import (
 	"sync"
+	"time"
 
 	"github.com/containerd/containerd/mount"
+	cdsnapshot "github.com/containerd/containerd/snapshot"
 	"github.com/moby/buildkit/cache/metadata"
 	"github.com/moby/buildkit/identity"
 	"github.com/moby/buildkit/util/flightcontrol"
@@ -134,7 +136,10 @@ func (cr *cacheRecord) Mount(ctx context.Context, readonly bool) ([]mount.Mount,
 	}
 	if cr.viewMount == nil { // TODO: handle this better
 		cr.view = identity.NewID()
-		m, err := cr.cm.Snapshotter.View(ctx, cr.view, cr.ID())
+		labels := map[string]string{
+			"containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339Nano),
+		}
+		m, err := cr.cm.Snapshotter.View(ctx, cr.view, cr.ID(), cdsnapshot.WithLabels(labels))
 		if err != nil {
 			cr.view = ""
 			return nil, errors.Wrapf(err, "failed to mount %s", cr.ID())

control/control_default.go

@@ -6,8 +6,8 @@ import (
 	"path/filepath"
 
 	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/diff"
 	"github.com/containerd/containerd/images"
-	"github.com/containerd/containerd/rootfs"
 	ctdsnapshot "github.com/containerd/containerd/snapshot"
 	"github.com/moby/buildkit/cache"
 	"github.com/moby/buildkit/cache/cacheimport"
@@ -31,8 +31,8 @@ import (
 type pullDeps struct {
 	Snapshotter  ctdsnapshot.Snapshotter
 	ContentStore content.Store
-	Applier      rootfs.Applier
-	Differ       rootfs.MountDiffer
+	Applier      diff.Differ
+	Differ       diff.Differ
 	Images       images.Store
 }
 

control/control_standalone.go

@@ -8,7 +8,7 @@ import (
 	"path/filepath"
 
 	"github.com/containerd/containerd/content/local"
-	"github.com/containerd/containerd/differ"
+	"github.com/containerd/containerd/diff/walking"
 	"github.com/containerd/containerd/mount"
 	"github.com/containerd/containerd/namespaces"
 	ctdsnapshot "github.com/containerd/containerd/snapshot"
@@ -55,7 +55,7 @@ func newStandalonePullDeps(root string) (*pullDeps, error) {
 		return nil, err
 	}
 
-	df, err := differ.NewWalkingDiff(c)
+	df, err := walking.NewWalkingDiff(c)
 	if err != nil {
 		return nil, err
 	}

examples/buildkit0/buildkit.go

@@ -17,8 +17,8 @@ type buildOpt struct {
 func main() {
 	var opt buildOpt
 	flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)")
-	flag.StringVar(&opt.containerd, "containerd", "v1.0.0-beta.1", "containerd version")
-	flag.StringVar(&opt.runc, "runc", "v1.0.0-rc4", "runc version")
+	flag.StringVar(&opt.containerd, "containerd", "v1.0.0-beta.3", "containerd version")
+	flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version")
 	flag.Parse()
 
 	bk := buildkit(opt)

examples/buildkit1/buildkit.go

@@ -17,8 +17,8 @@ type buildOpt struct {
 func main() {
 	var opt buildOpt
 	flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)")
-	flag.StringVar(&opt.containerd, "containerd", "v1.0.0-beta.1", "containerd version")
-	flag.StringVar(&opt.runc, "runc", "v1.0.0-rc4", "runc version")
+	flag.StringVar(&opt.containerd, "containerd", "v1.0.0-beta.3", "containerd version")
+	flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version")
 	flag.Parse()
 
 	bk := buildkit(opt)

examples/buildkit2/buildkit.go

@@ -17,8 +17,8 @@ type buildOpt struct {
 func main() {
 	var opt buildOpt
 	flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)")
-	flag.StringVar(&opt.containerd, "containerd", "v1.0.0-beta.1", "containerd version")
-	flag.StringVar(&opt.runc, "runc", "v1.0.0-rc4", "runc version")
+	flag.StringVar(&opt.containerd, "containerd", "v1.0.0-beta.3", "containerd version")
+	flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version")
 	flag.Parse()
 
 	bk := buildkit(opt)

examples/buildkit3/buildkit.go

@@ -18,8 +18,8 @@ type buildOpt struct {
 func main() {
 	var opt buildOpt
 	flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)")
-	flag.StringVar(&opt.containerd, "containerd", "v1.0.0-beta.1", "containerd version")
-	flag.StringVar(&opt.runc, "runc", "v1.0.0-rc4", "runc version")
+	flag.StringVar(&opt.containerd, "containerd", "ab67fd50dcb4c5b7ddb0c482c1ee542b0ed87e33", "containerd version")
+	flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version")
 	flag.StringVar(&opt.buildkit, "buildkit", "master", "buildkit version")
 	flag.Parse()
 

exporter/containerimage/export.go

@@ -7,9 +7,9 @@ import (
 	"time"
 
 	"github.com/containerd/containerd/content"
+	"github.com/containerd/containerd/diff"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/images"
-	"github.com/containerd/containerd/rootfs"
 	"github.com/docker/distribution"
 	"github.com/docker/distribution/manifest/schema2"
 	"github.com/moby/buildkit/cache"
@@ -38,7 +38,7 @@ type Opt struct {
 	SessionManager *session.Manager
 	Snapshotter    snapshot.Snapshotter
 	ContentStore   content.Store
-	Differ         rootfs.MountDiffer
+	Differ         diff.Differ
 	Images         images.Store
 }
 

hack/dockerfiles/test.Dockerfile

@@ -1,5 +1,5 @@
-ARG RUNC_VERSION=e775f0fba3ea329b8b766451c892c41a3d49594d
-ARG CONTAINERD_VERSION=d1e11f17ec7b325f89608dd46c128300b8727d50
+ARG RUNC_VERSION=74a17296470088de3805e138d3d87c62e613dfc4
+ARG CONTAINERD_VERSION=v1.0.0-beta.3
 ARG BUILDKIT_TARGET=standalone
 
 FROM golang:1.9-alpine AS gobuild-base

snapshot/localmounter.go

@@ -48,7 +48,7 @@ func (lm *localMounter) Mount() (string, error) {
 		return "", errors.Wrap(err, "failed to create temp dir")
 	}
 
-	if err := mount.MountAll(lm.m, dir); err != nil {
+	if err := mount.All(lm.m, dir); err != nil {
 		os.RemoveAll(dir)
 		return "", err
 	}