Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

bump up containerd (v1.0.0-beta.3) #155

Merged
merged 1 commit into from Nov 13, 2017
Merged

bump up containerd (v1.0.0-beta.3) #155

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 1 addition & 1 deletion README.md
View file
Open in desktop
Expand Up @@ -123,7 +123,7 @@ The `tonistiigi/buildkit:standalone` image can be built locally using the Docker

#### Supported runc version

During development buildkit is tested with the version of runc that is being used by the containerd repository. Please refer to [runc.md](https://github.com/containerd/containerd/blob/d1e11f17ec7b325f89608dd46c128300b8727d50/RUNC.md) for more information.
During development buildkit is tested with the version of runc that is being used by the containerd repository. Please refer to [runc.md](https://github.com/containerd/containerd/blob/v1.0.0-beta.3/RUNC.md) for more information.


#### Contributing
Expand Down
8 changes: 5 additions & 3 deletions cache/blobs/blobs.go
View file
Open in desktop
Expand Up @@ -3,8 +3,8 @@ package blobs
import (
gocontext "context"

"github.com/containerd/containerd/diff"
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/rootfs"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/snapshot"
"github.com/moby/buildkit/util/flightcontrol"
Expand All @@ -27,7 +27,7 @@ type blobmapper interface {
SetBlob(ctx gocontext.Context, key string, diffID, blob digest.Digest) error
}

func GetDiffPairs(ctx context.Context, snapshotter snapshot.Snapshotter, differ rootfs.MountDiffer, ref cache.ImmutableRef) ([]DiffPair, error) {
func GetDiffPairs(ctx context.Context, snapshotter snapshot.Snapshotter, differ diff.Differ, ref cache.ImmutableRef) ([]DiffPair, error) {
blobmap, ok := snapshotter.(blobmapper)
if !ok {
return nil, errors.Errorf("image exporter requires snapshotter with blobs mapping support")
Expand Down Expand Up @@ -71,7 +71,9 @@ func GetDiffPairs(ctx context.Context, snapshotter snapshot.Snapshotter, differ
if err != nil {
return nil, err
}
descr, err := differ.DiffMounts(ctx, lower, upper, ocispec.MediaTypeImageLayer, ref.ID())
descr, err := differ.DiffMounts(ctx, lower, upper,
diff.WithMediaType(ocispec.MediaTypeImageLayer),
diff.WithReference(ref.ID()))
if err != nil {
return nil, err
}
Expand Down
4 changes: 2 additions & 2 deletions cache/cacheimport/export.go
View file
Open in desktop
Expand Up @@ -6,8 +6,8 @@ import (
"encoding/json"

"github.com/containerd/containerd/content"
"github.com/containerd/containerd/diff"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/rootfs"
"github.com/docker/distribution/manifest"
"github.com/docker/distribution/manifest/schema2"
"github.com/moby/buildkit/cache"
Expand Down Expand Up @@ -38,7 +38,7 @@ type CacheRecord struct {
type ExporterOpt struct {
Snapshotter snapshot.Snapshotter
ContentStore content.Store
Differ rootfs.MountDiffer
Differ diff.Differ
SessionManager *session.Manager
}

Expand Down
18 changes: 14 additions & 4 deletions cache/cacheimport/import.go
View file
Open in desktop
Expand Up @@ -7,9 +7,11 @@ import (
"time"

"github.com/containerd/containerd/content"
"github.com/containerd/containerd/diff"
"github.com/containerd/containerd/remotes"
"github.com/containerd/containerd/remotes/docker"
"github.com/containerd/containerd/rootfs"
cdsnapshot "github.com/containerd/containerd/snapshot"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/blobs"
"github.com/moby/buildkit/client"
Expand All @@ -30,7 +32,7 @@ type ImportOpt struct {
SessionManager *session.Manager
ContentStore content.Store
Snapshotter snapshot.Snapshotter
Applier rootfs.Applier
Applier diff.Differ
CacheAccessor cache.Accessor
}

Expand Down Expand Up @@ -263,10 +265,18 @@ func (ii *importInfo) unpack(ctx context.Context, dpairs []blobs.DiffPair) (stri
return "", err
}

chainID, err := rootfs.ApplyLayers(ctx, layers, ii.opt.Snapshotter, ii.opt.Applier)
if err != nil {
return "", err
var chain []digest.Digest
for _, layer := range layers {
labels := map[string]string{
"containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339Nano),
"containerd.io/uncompressed": layer.Diff.Digest.String(),
}
if _, err := rootfs.ApplyLayer(ctx, layer, chain, ii.opt.Snapshotter, ii.opt.Applier, cdsnapshot.WithLabels(labels)); err != nil {
return "", err
}
chain = append(chain, layer.Diff.Digest)
}
chainID := identity.ChainID(chain)

if err := ii.fillBlobMapping(ctx, layers); err != nil {
return "", err
Expand Down
20 changes: 12 additions & 8 deletions cache/contenthash/checksum_test.go
View file
Open in desktop
Expand Up @@ -33,7 +33,9 @@ func TestChecksumBasicFile(t *testing.T) {
require.NoError(t, err)
defer os.RemoveAll(tmpdir)

cm := setupCacheManager(t, tmpdir)
snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -180,7 +182,9 @@ func TestHandleChange(t *testing.T) {
require.NoError(t, err)
defer os.RemoveAll(tmpdir)

cm := setupCacheManager(t, tmpdir)
snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -255,7 +259,9 @@ func TestPersistence(t *testing.T) {
require.NoError(t, err)
defer os.RemoveAll(tmpdir)

cm := setupCacheManager(t, tmpdir)
snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := setupCacheManager(t, tmpdir, snapshotter)
defer cm.Close()

ch := []string{
Expand Down Expand Up @@ -289,9 +295,10 @@ func TestPersistence(t *testing.T) {

time.Sleep(100 * time.Millisecond) // saving happens on the background

// we can't close snapshotter and open it twice (especially, its internal boltdb store)
cm.Close()
getDefaultManager().lru.Purge()
cm = setupCacheManager(t, tmpdir)
cm = setupCacheManager(t, tmpdir, snapshotter)
defer cm.Close()

ref, err = cm.Get(context.TODO(), id)
Expand Down Expand Up @@ -324,10 +331,7 @@ func createRef(t *testing.T, cm cache.Manager, files []string) cache.ImmutableRe
return ref
}

func setupCacheManager(t *testing.T, tmpdir string) cache.Manager {
snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)

func setupCacheManager(t *testing.T, tmpdir string, snapshotter snapshot.Snapshotter) cache.Manager {
md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db"))
require.NoError(t, err)

Expand Down
5 changes: 4 additions & 1 deletion cache/manager.go
View file
Open in desktop
Expand Up @@ -188,7 +188,10 @@ func (cm *cacheManager) New(ctx context.Context, s ImmutableRef, opts ...RefOpti
parentID = parent.ID()
}

if _, err := cm.Snapshotter.Prepare(ctx, id, parentID); err != nil {
labels := map[string]string{
"containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339Nano),
}
if _, err := cm.Snapshotter.Prepare(ctx, id, parentID, cdsnapshot.WithLabels(labels)); err != nil {
if parent != nil {
parent.Release(context.TODO())
}
Expand Down
18 changes: 10 additions & 8 deletions cache/manager_test.go
View file
Open in desktop
Expand Up @@ -24,7 +24,9 @@ func TestManager(t *testing.T) {
require.NoError(t, err)
defer os.RemoveAll(tmpdir)

cm := getCacheManager(t, tmpdir)
snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := getCacheManager(t, tmpdir, snapshotter)

_, err = cm.Get(ctx, "foobar")
require.Error(t, err)
Expand Down Expand Up @@ -132,7 +134,9 @@ func TestLazyCommit(t *testing.T) {
require.NoError(t, err)
defer os.RemoveAll(tmpdir)

cm := getCacheManager(t, tmpdir)
snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)
cm := getCacheManager(t, tmpdir, snapshotter)

active, err := cm.New(ctx, nil, CachePolicyRetain)
require.NoError(t, err)
Expand Down Expand Up @@ -213,7 +217,8 @@ func TestLazyCommit(t *testing.T) {
err = cm.Close()
require.NoError(t, err)

cm = getCacheManager(t, tmpdir)
// we can't close snapshotter and open it twice (especially, its internal boltdb store)
cm = getCacheManager(t, tmpdir, snapshotter)

snap2, err = cm.Get(ctx, snap.ID())
require.NoError(t, err)
Expand All @@ -234,7 +239,7 @@ func TestLazyCommit(t *testing.T) {
err = cm.Close()
require.NoError(t, err)

cm = getCacheManager(t, tmpdir)
cm = getCacheManager(t, tmpdir, snapshotter)

snap2, err = cm.Get(ctx, snap.ID())
require.NoError(t, err)
Expand All @@ -250,10 +255,7 @@ func TestLazyCommit(t *testing.T) {
require.Equal(t, errNotFound, errors.Cause(err))
}

func getCacheManager(t *testing.T, tmpdir string) Manager {
snapshotter, err := naive.NewSnapshotter(filepath.Join(tmpdir, "snapshots"))
require.NoError(t, err)

func getCacheManager(t *testing.T, tmpdir string, snapshotter snapshot.Snapshotter) Manager {
md, err := metadata.NewStore(filepath.Join(tmpdir, "metadata.db"))
require.NoError(t, err)

Expand Down
7 changes: 6 additions & 1 deletion cache/refs.go
View file
Open in desktop
Expand Up @@ -2,8 +2,10 @@ package cache

import (
"sync"
"time"

"github.com/containerd/containerd/mount"
cdsnapshot "github.com/containerd/containerd/snapshot"
"github.com/moby/buildkit/cache/metadata"
"github.com/moby/buildkit/identity"
"github.com/moby/buildkit/util/flightcontrol"
Expand Down Expand Up @@ -134,7 +136,10 @@ func (cr *cacheRecord) Mount(ctx context.Context, readonly bool) ([]mount.Mount,
}
if cr.viewMount == nil { // TODO: handle this better
cr.view = identity.NewID()
m, err := cr.cm.Snapshotter.View(ctx, cr.view, cr.ID())
labels := map[string]string{
"containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339Nano),
}
m, err := cr.cm.Snapshotter.View(ctx, cr.view, cr.ID(), cdsnapshot.WithLabels(labels))
if err != nil {
cr.view = ""
return nil, errors.Wrapf(err, "failed to mount %s", cr.ID())
Expand Down
6 changes: 3 additions & 3 deletions control/control_default.go
View file
Open in desktop
Expand Up @@ -6,8 +6,8 @@ import (
"path/filepath"

"github.com/containerd/containerd/content"
"github.com/containerd/containerd/diff"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/rootfs"
ctdsnapshot "github.com/containerd/containerd/snapshot"
"github.com/moby/buildkit/cache"
"github.com/moby/buildkit/cache/cacheimport"
Expand All @@ -31,8 +31,8 @@ import (
type pullDeps struct {
Snapshotter ctdsnapshot.Snapshotter
ContentStore content.Store
Applier rootfs.Applier
Differ rootfs.MountDiffer
Applier diff.Differ
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

😢 I liked the small interfaces. I guess it doesn't make sense to keep them different in buildkit anymore as well then. (Can be follow-up).

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dmcgowan WDYT about splitting Applier from Differ again?

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Are there many places we need both interfaces together? We can split and have a combined interface as well.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@dmcgowan This is the place that creates all the dependencies that are sent to other objects, so nothing actually uses them in here. But, for example, before the pull component only depended on the apply function and image exporter only on the differ. Now there is no separation.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

opened containerd/containerd#1746

Differ diff.Differ
Images images.Store
}

Expand Down
4 changes: 2 additions & 2 deletions control/control_standalone.go
View file
Open in desktop
Expand Up @@ -8,7 +8,7 @@ import (
"path/filepath"

"github.com/containerd/containerd/content/local"
"github.com/containerd/containerd/differ"
"github.com/containerd/containerd/diff/walking"
"github.com/containerd/containerd/mount"
"github.com/containerd/containerd/namespaces"
ctdsnapshot "github.com/containerd/containerd/snapshot"
Expand Down Expand Up @@ -55,7 +55,7 @@ func newStandalonePullDeps(root string) (*pullDeps, error) {
return nil, err
}

df, err := differ.NewWalkingDiff(c)
df, err := walking.NewWalkingDiff(c)
if err != nil {
return nil, err
}
Expand Down
4 changes: 2 additions & 2 deletions examples/buildkit0/buildkit.go
View file
Open in desktop
Expand Up @@ -17,8 +17,8 @@ type buildOpt struct {
func main() {
var opt buildOpt
flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)")
flag.StringVar(&opt.containerd, "containerd", "v1.0.0-beta.1", "containerd version")
flag.StringVar(&opt.runc, "runc", "v1.0.0-rc4", "runc version")
flag.StringVar(&opt.containerd, "containerd", "v1.0.0-beta.3", "containerd version")
flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version")
flag.Parse()

bk := buildkit(opt)
Expand Down
4 changes: 2 additions & 2 deletions examples/buildkit1/buildkit.go
View file
Open in desktop
Expand Up @@ -17,8 +17,8 @@ type buildOpt struct {
func main() {
var opt buildOpt
flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)")
flag.StringVar(&opt.containerd, "containerd", "v1.0.0-beta.1", "containerd version")
flag.StringVar(&opt.runc, "runc", "v1.0.0-rc4", "runc version")
flag.StringVar(&opt.containerd, "containerd", "v1.0.0-beta.3", "containerd version")
flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version")
flag.Parse()

bk := buildkit(opt)
Expand Down
4 changes: 2 additions & 2 deletions examples/buildkit2/buildkit.go
View file
Open in desktop
Expand Up @@ -17,8 +17,8 @@ type buildOpt struct {
func main() {
var opt buildOpt
flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)")
flag.StringVar(&opt.containerd, "containerd", "v1.0.0-beta.1", "containerd version")
flag.StringVar(&opt.runc, "runc", "v1.0.0-rc4", "runc version")
flag.StringVar(&opt.containerd, "containerd", "v1.0.0-beta.3", "containerd version")
flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version")
flag.Parse()

bk := buildkit(opt)
Expand Down
4 changes: 2 additions & 2 deletions examples/buildkit3/buildkit.go
View file
Open in desktop
Expand Up @@ -18,8 +18,8 @@ type buildOpt struct {
func main() {
var opt buildOpt
flag.StringVar(&opt.target, "target", "containerd", "target (standalone, containerd)")
flag.StringVar(&opt.containerd, "containerd", "v1.0.0-beta.1", "containerd version")
flag.StringVar(&opt.runc, "runc", "v1.0.0-rc4", "runc version")
flag.StringVar(&opt.containerd, "containerd", "ab67fd50dcb4c5b7ddb0c482c1ee542b0ed87e33", "containerd version")
flag.StringVar(&opt.runc, "runc", "74a17296470088de3805e138d3d87c62e613dfc4", "runc version")
flag.StringVar(&opt.buildkit, "buildkit", "master", "buildkit version")
flag.Parse()

Expand Down
4 changes: 2 additions & 2 deletions exporter/containerimage/export.go
View file
Open in desktop
Expand Up @@ -7,9 +7,9 @@ import (
"time"

"github.com/containerd/containerd/content"
"github.com/containerd/containerd/diff"
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/images"
"github.com/containerd/containerd/rootfs"
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema2"
"github.com/moby/buildkit/cache"
Expand Down Expand Up @@ -38,7 +38,7 @@ type Opt struct {
SessionManager *session.Manager
Snapshotter snapshot.Snapshotter
ContentStore content.Store
Differ rootfs.MountDiffer
Differ diff.Differ
Images images.Store
}

Expand Down
4 changes: 2 additions & 2 deletions hack/dockerfiles/test.Dockerfile
View file
Open in desktop
@@ -1,5 +1,5 @@
ARG RUNC_VERSION=e775f0fba3ea329b8b766451c892c41a3d49594d
ARG CONTAINERD_VERSION=d1e11f17ec7b325f89608dd46c128300b8727d50
ARG RUNC_VERSION=74a17296470088de3805e138d3d87c62e613dfc4
ARG CONTAINERD_VERSION=v1.0.0-beta.3
ARG BUILDKIT_TARGET=standalone

FROM golang:1.9-alpine AS gobuild-base
Expand Down
2 changes: 1 addition & 1 deletion snapshot/localmounter.go
View file
Open in desktop
Expand Up @@ -48,7 +48,7 @@ func (lm *localMounter) Mount() (string, error) {
return "", errors.Wrap(err, "failed to create temp dir")
}

if err := mount.MountAll(lm.m, dir); err != nil {
if err := mount.All(lm.m, dir); err != nil {
os.RemoveAll(dir)
return "", err
}
Expand Down