Skip to content

Commit

Permalink
Merge pull request #439 from AkihiroSuda/vendor-containerd-20180606
Browse files Browse the repository at this point in the history 
update containerd library
  • Loading branch information
@tonistiigi
tonistiigi committed Jun 8, 2018
2 parents cf4cc2d + 47a00b8 commit 7943598
Show file tree
Hide file tree
Showing 64 changed files with 2,108 additions and 299 deletions.
20 changes: 12 additions & 8 deletions cache/remotecache/export.go
View file
Expand Up @@ -77,20 +77,20 @@ func (ce *CacheExporter) Finalize(ctx context.Context, cc *v1.CacheChains, targe
return err
}
dgst := digest.FromBytes(dt)

desc := ocispec.Descriptor{
Digest: dgst,
Size: int64(len(dt)),
MediaType: v1.CacheConfigMediaTypeV0,
}
configDone := oneOffProgress(ctx, fmt.Sprintf("writing config %s", dgst))
buf := contentutil.NewBuffer()
if err := content.WriteBlob(ctx, buf, dgst.String(), bytes.NewReader(dt), int64(len(dt)), dgst); err != nil {
if err := content.WriteBlob(ctx, buf, dgst.String(), bytes.NewReader(dt), desc); err != nil {
return configDone(errors.Wrap(err, "error writing config blob"))
}
configDone(nil)

mp.Add(dgst, buf)
mfst.Manifests = append(mfst.Manifests, ocispec.Descriptor{
MediaType: v1.CacheConfigMediaTypeV0,
Size: int64(len(dt)),
Digest: dgst,
})
mfst.Manifests = append(mfst.Manifests, desc)

dt, err = json.Marshal(mfst)
if err != nil {
Expand All @@ -99,8 +99,12 @@ func (ce *CacheExporter) Finalize(ctx context.Context, cc *v1.CacheChains, targe
dgst = digest.FromBytes(dt)

buf = contentutil.NewBuffer()
desc = ocispec.Descriptor{
Digest: dgst,
Size: int64(len(dt)),
}
mfstDone := oneOffProgress(ctx, fmt.Sprintf("writing manifest %s", dgst))
if err := content.WriteBlob(ctx, buf, dgst.String(), bytes.NewReader(dt), int64(len(dt)), dgst); err != nil {
if err := content.WriteBlob(ctx, buf, dgst.String(), bytes.NewReader(dt), desc); err != nil {
return mfstDone(errors.Wrap(err, "error writing manifest blob"))
}
mfstDone(nil)
Expand Down
4 changes: 2 additions & 2 deletions cache/remotecache/import.go
View file
Expand Up @@ -73,7 +73,7 @@ func (ci *CacheImporter) Resolve(ctx context.Context, ref string) (solver.CacheM
return nil, err
}

dt, err := content.ReadBlob(ctx, b, desc.Digest)
dt, err := content.ReadBlob(ctx, b, desc)
if err != nil {
return nil, err
}
Expand Down Expand Up @@ -106,7 +106,7 @@ func (ci *CacheImporter) Resolve(ctx context.Context, ref string) (solver.CacheM
return nil, err
}

dt, err = content.ReadBlob(ctx, b, configDesc.Digest)
dt, err = content.ReadBlob(ctx, b, configDesc)
if err != nil {
return nil, err
}
Expand Down
8 changes: 4 additions & 4 deletions client/client_test.go
View file
Expand Up @@ -553,7 +553,7 @@ func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
desc, err := img.Config(ctx)
require.NoError(t, err)

dt, err = content.ReadBlob(ctx, img.ContentStore(), desc.Digest)
dt, err = content.ReadBlob(ctx, img.ContentStore(), desc)
require.NoError(t, err)

var ociimg ocispec.Image
Expand Down Expand Up @@ -584,7 +584,7 @@ func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
require.True(t, ociimg.History[1].EmptyLayer)
require.False(t, ociimg.History[2].EmptyLayer)

dt, err = content.ReadBlob(ctx, img.ContentStore(), img.Target().Digest)
dt, err = content.ReadBlob(ctx, img.ContentStore(), img.Target())
require.NoError(t, err)

var mfst schema2.Manifest
Expand All @@ -594,7 +594,7 @@ func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
require.Equal(t, schema2.MediaTypeManifest, mfst.MediaType)
require.Equal(t, 2, len(mfst.Layers))

dt, err = content.ReadBlob(ctx, img.ContentStore(), mfst.Layers[0].Digest)
dt, err = content.ReadBlob(ctx, img.ContentStore(), ocispec.Descriptor{Digest: mfst.Layers[0].Digest})
require.NoError(t, err)

m, err := readTarToMap(dt, true)
Expand All @@ -617,7 +617,7 @@ func testBuildPushAndValidate(t *testing.T, sb integration.Sandbox) {
_, ok = m["foo/sub/baz"]
require.False(t, ok)

dt, err = content.ReadBlob(ctx, img.ContentStore(), mfst.Layers[1].Digest)
dt, err = content.ReadBlob(ctx, img.ContentStore(), ocispec.Descriptor{Digest: mfst.Layers[1].Digest})
require.NoError(t, err)

m, err = readTarToMap(dt, true)
Expand Down
12 changes: 10 additions & 2 deletions exporter/containerimage/writer.go
View file
Expand Up @@ -103,16 +103,24 @@ func (ic *ImageWriter) Commit(ctx context.Context, ref cache.ImmutableRef, confi
}

mfstDigest := digest.FromBytes(mfstJSON)
mfstDesc := ocispec.Descriptor{
Digest: mfstDigest,
Size: int64(len(mfstJSON)),
}
mfstDone := oneOffProgress(ctx, "exporting manifest "+mfstDigest.String())

if err := content.WriteBlob(ctx, ic.opt.ContentStore, mfstDigest.String(), bytes.NewReader(mfstJSON), int64(len(mfstJSON)), mfstDigest, content.WithLabels(labels)); err != nil {
if err := content.WriteBlob(ctx, ic.opt.ContentStore, mfstDigest.String(), bytes.NewReader(mfstJSON), mfstDesc, content.WithLabels(labels)); err != nil {
return nil, mfstDone(errors.Wrapf(err, "error writing manifest blob %s", mfstDigest))
}
mfstDone(nil)

configDesc := ocispec.Descriptor{
Digest: configDigest,
Size: int64(len(config)),
}
configDone := oneOffProgress(ctx, "exporting config "+configDigest.String())

if err := content.WriteBlob(ctx, ic.opt.ContentStore, configDigest.String(), bytes.NewReader(config), int64(len(config)), configDigest); err != nil {
if err := content.WriteBlob(ctx, ic.opt.ContentStore, configDigest.String(), bytes.NewReader(config), configDesc); err != nil {
return nil, configDone(errors.Wrap(err, "error writing config blob"))
}
configDone(nil)
Expand Down
14 changes: 7 additions & 7 deletions frontend/dockerfile/dockerfile_test.go
View file
Expand Up @@ -238,7 +238,7 @@ ENTRYPOINT my entrypoint
desc, err := img.Config(ctx, ctr.ContentStore(), platforms.Default())
require.NoError(t, err)

dt, err := content.ReadBlob(ctx, ctr.ContentStore(), desc.Digest)
dt, err := content.ReadBlob(ctx, ctr.ContentStore(), desc)
require.NoError(t, err)

var ociimg ocispec.Image
Expand Down Expand Up @@ -329,7 +329,7 @@ COPY foo .
desc, err := img.Config(ctx, ctr.ContentStore(), platforms.Default())
require.NoError(t, err)

dt, err := content.ReadBlob(ctx, ctr.ContentStore(), desc.Digest)
dt, err := content.ReadBlob(ctx, ctr.ContentStore(), desc)
require.NoError(t, err)

var ociimg ocispec.Image
Expand Down Expand Up @@ -828,7 +828,7 @@ ENV foo=bar
desc, err := img.Config(ctx, client.ContentStore(), platforms.Default())
require.NoError(t, err)

dt, err := content.ReadBlob(ctx, client.ContentStore(), desc.Digest)
dt, err := content.ReadBlob(ctx, client.ContentStore(), desc)
require.NoError(t, err)

var ociimg ocispec.Image
Expand Down Expand Up @@ -911,7 +911,7 @@ EXPOSE 5000
desc, err := img.Config(ctx, client.ContentStore(), platforms.Default())
require.NoError(t, err)

dt, err := content.ReadBlob(ctx, client.ContentStore(), desc.Digest)
dt, err := content.ReadBlob(ctx, client.ContentStore(), desc)
require.NoError(t, err)

var ociimg ocispec.Image
Expand Down Expand Up @@ -1091,7 +1091,7 @@ RUN ["ls"]
desc, err := img.Config(ctx, client.ContentStore(), platforms.Default())
require.NoError(t, err)

dt, err := content.ReadBlob(ctx, client.ContentStore(), desc.Digest)
dt, err := content.ReadBlob(ctx, client.ContentStore(), desc)
require.NoError(t, err)

var ociimg ocispec.Image
Expand Down Expand Up @@ -1196,7 +1196,7 @@ USER nobody
desc, err := img.Config(ctx, client.ContentStore(), platforms.Default())
require.NoError(t, err)

dt, err = content.ReadBlob(ctx, client.ContentStore(), desc.Digest)
dt, err = content.ReadBlob(ctx, client.ContentStore(), desc)
require.NoError(t, err)

var ociimg ocispec.Image
Expand Down Expand Up @@ -1769,7 +1769,7 @@ LABEL foo=bar
desc, err := img.Config(ctx, client.ContentStore(), platforms.Default())
require.NoError(t, err)

dt, err := content.ReadBlob(ctx, client.ContentStore(), desc.Digest)
dt, err := content.ReadBlob(ctx, client.ContentStore(), desc)
require.NoError(t, err)

var ociimg ocispec.Image
Expand Down
2 changes: 1 addition & 1 deletion hack/update-vendor
View file
Expand Up @@ -3,7 +3,7 @@
set -eu -o pipefail -x

iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
docker build --build-arg VNDR_VERSION=15dc0965b7ba6dd78ae741e6fda33f23207033c4 --iidfile $iidfile -f ./hack/dockerfiles/vendor.Dockerfile --force-rm .
docker build --build-arg VNDR_VERSION=1fc68ee0c852556a9ed53cbde16247033f104111 --iidfile $iidfile -f ./hack/dockerfiles/vendor.Dockerfile --force-rm .
iid=$(cat $iidfile)
cid=$(docker create $iid noop)
rm -rf ./vendor
Expand Down
2 changes: 1 addition & 1 deletion hack/validate-vendor
View file
Expand Up @@ -3,7 +3,7 @@
set -eu -o pipefail -x

iidfile=$(mktemp -t docker-iidfile.XXXXXXXXXX)
docker build --build-arg VNDR_VERSION=15dc0965b7ba6dd78ae741e6fda33f23207033c4 --iidfile $iidfile -f ./hack/dockerfiles/vendor.Dockerfile --force-rm .
docker build --build-arg VNDR_VERSION=1fc68ee0c852556a9ed53cbde16247033f104111 --iidfile $iidfile -f ./hack/dockerfiles/vendor.Dockerfile --force-rm .
iid=$(cat $iidfile)
diffs="$(docker run --rm $iid git status --porcelain -- vendor 2>/dev/null)"
if [ "$diffs" ]; then
Expand Down
29 changes: 18 additions & 11 deletions snapshot/containerd/content.go
View file
Expand Up @@ -8,6 +8,7 @@ import (
"github.com/containerd/containerd/errdefs"
"github.com/containerd/containerd/namespaces"
"github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)

type garbageCollectFn func(context.Context) error
Expand Down Expand Up @@ -65,28 +66,34 @@ func (c *nsContent) Abort(ctx context.Context, ref string) error {
return c.Store.Abort(ctx, ref)
}

func (c *nsContent) ReaderAt(ctx context.Context, dgst digest.Digest) (content.ReaderAt, error) {
func (c *nsContent) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content.ReaderAt, error) {
ctx = namespaces.WithNamespace(ctx, c.ns)
return c.Store.ReaderAt(ctx, dgst)
return c.Store.ReaderAt(ctx, desc)
}

func (c *nsContent) Writer(ctx context.Context, ref string, size int64, expected digest.Digest) (content.Writer, error) {
return c.writer(ctx, ref, size, expected, 3)
func (c *nsContent) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
return c.writer(ctx, 3, opts...)
}

func (c *nsContent) writer(ctx context.Context, ref string, size int64, expected digest.Digest, retries int) (content.Writer, error) {
func (c *nsContent) writer(ctx context.Context, retries int, opts ...content.WriterOpt) (content.Writer, error) {
var wOpts content.WriterOpts
for _, opt := range opts {
if err := opt(&wOpts); err != nil {
return nil, err
}
}
ctx = namespaces.WithNamespace(ctx, c.ns)
w, err := c.Store.Writer(ctx, ref, size, expected)
w, err := c.Store.Writer(ctx, opts...)
if err != nil {
if errdefs.IsAlreadyExists(err) && expected != "" && retries > 0 {
if errdefs.IsAlreadyExists(err) && wOpts.Desc.Digest != "" && retries > 0 {
_, err2 := c.Update(ctx, content.Info{
Digest: expected,
Digest: wOpts.Desc.Digest,
Labels: map[string]string{
"containerd.io/gc.root": time.Now().UTC().Format(time.RFC3339Nano),
},
}, "labels.containerd.io/gc.root")
if err2 != nil {
return c.writer(ctx, ref, size, expected, retries-1)
return c.writer(ctx, retries-1, opts...)
}
}
}
Expand All @@ -100,8 +107,8 @@ type noGCWriter struct {
content.Writer
}

func (cs *noGCContentStore) Writer(ctx context.Context, ref string, size int64, expected digest.Digest) (content.Writer, error) {
w, err := cs.Store.Writer(ctx, ref, size, expected)
func (cs *noGCContentStore) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
w, err := cs.Store.Writer(ctx, opts...)
return &noGCWriter{w}, err
}

Expand Down
21 changes: 14 additions & 7 deletions util/contentutil/buffer.go
View file
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
)

Expand All @@ -33,27 +34,33 @@ type buffer struct {
refs map[string]struct{}
}

func (b *buffer) Writer(ctx context.Context, ref string, size int64, expected digest.Digest) (content.Writer, error) {
func (b *buffer) Writer(ctx context.Context, opts ...content.WriterOpt) (content.Writer, error) {
var wOpts content.WriterOpts
for _, opt := range opts {
if err := opt(&wOpts); err != nil {
return nil, err
}
}
b.mu.Lock()
if _, ok := b.refs[ref]; ok {
return nil, errors.Wrapf(errdefs.ErrUnavailable, "ref %s locked", ref)
if _, ok := b.refs[wOpts.Ref]; ok {
return nil, errors.Wrapf(errdefs.ErrUnavailable, "ref %s locked", wOpts.Ref)
}
b.mu.Unlock()
return &bufferedWriter{
main: b,
digester: digest.Canonical.Digester(),
buffer: bytes.NewBuffer(nil),
expected: expected,
expected: wOpts.Desc.Digest,
releaseRef: func() {
b.mu.Lock()
delete(b.refs, ref)
delete(b.refs, wOpts.Ref)
b.mu.Unlock()
},
}, nil
}

func (b *buffer) ReaderAt(ctx context.Context, dgst digest.Digest) (content.ReaderAt, error) {
r, err := b.getBytesReader(ctx, dgst)
func (b *buffer) ReaderAt(ctx context.Context, desc ocispec.Descriptor) (content.ReaderAt, error) {
r, err := b.getBytesReader(ctx, desc.Digest)
if err != nil {
return nil, err
}
Expand Down
19 changes: 10 additions & 9 deletions util/contentutil/buffer_test.go
View file
Expand Up @@ -9,6 +9,7 @@ import (
"github.com/containerd/containerd/content"
"github.com/containerd/containerd/errdefs"
digest "github.com/opencontainers/go-digest"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
)
Expand All @@ -19,26 +20,26 @@ func TestReadWrite(t *testing.T) {

b := NewBuffer()

err := content.WriteBlob(ctx, b, "foo", bytes.NewBuffer([]byte("foo0")), -1, "")
err := content.WriteBlob(ctx, b, "foo", bytes.NewBuffer([]byte("foo0")), ocispec.Descriptor{Size: -1})
require.NoError(t, err)

err = content.WriteBlob(ctx, b, "foo", bytes.NewBuffer([]byte("foo1")), 4, "")
err = content.WriteBlob(ctx, b, "foo", bytes.NewBuffer([]byte("foo1")), ocispec.Descriptor{Size: 4})
require.NoError(t, err)

err = content.WriteBlob(ctx, b, "foo", bytes.NewBuffer([]byte("foo2")), 3, "")
err = content.WriteBlob(ctx, b, "foo", bytes.NewBuffer([]byte("foo2")), ocispec.Descriptor{Size: 3})
require.Error(t, err)

err = content.WriteBlob(ctx, b, "foo", bytes.NewBuffer([]byte("foo3")), -1, digest.FromBytes([]byte("foo4")))
err = content.WriteBlob(ctx, b, "foo", bytes.NewBuffer([]byte("foo3")), ocispec.Descriptor{Size: -1, Digest: digest.FromBytes([]byte("foo4"))})
require.Error(t, err)

err = content.WriteBlob(ctx, b, "foo", bytes.NewBuffer([]byte("foo4")), -1, digest.FromBytes([]byte("foo4")))
err = content.WriteBlob(ctx, b, "foo", bytes.NewBuffer([]byte("foo4")), ocispec.Descriptor{Size: -1, Digest: digest.FromBytes([]byte("foo4"))})
require.NoError(t, err)

dt, err := content.ReadBlob(ctx, b, digest.FromBytes([]byte("foo1")))
dt, err := content.ReadBlob(ctx, b, ocispec.Descriptor{Digest: digest.FromBytes([]byte("foo1"))})
require.NoError(t, err)
require.Equal(t, string(dt), "foo1")

_, err = content.ReadBlob(ctx, b, digest.FromBytes([]byte("foo3")))
_, err = content.ReadBlob(ctx, b, ocispec.Descriptor{Digest: digest.FromBytes([]byte("foo3"))})
require.Error(t, err)
require.Equal(t, errors.Cause(err), errdefs.ErrNotFound)
}
Expand All @@ -49,10 +50,10 @@ func TestReaderAt(t *testing.T) {

b := NewBuffer()

err := content.WriteBlob(ctx, b, "foo", bytes.NewBuffer([]byte("foobar")), -1, "")
err := content.WriteBlob(ctx, b, "foo", bytes.NewBuffer([]byte("foobar")), ocispec.Descriptor{Size: -1})
require.NoError(t, err)

rdr, err := b.ReaderAt(ctx, digest.FromBytes([]byte("foobar")))
rdr, err := b.ReaderAt(ctx, ocispec.Descriptor{Digest: digest.FromBytes([]byte("foobar"))})
require.NoError(t, err)

require.Equal(t, int64(6), rdr.Size())
Expand Down
2 changes: 1 addition & 1 deletion util/contentutil/copy.go
View file
Expand Up @@ -21,7 +21,7 @@ type localFetcher struct {
}

func (f *localFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, error) {
r, err := f.Provider.ReaderAt(ctx, desc.Digest)
r, err := f.Provider.ReaderAt(ctx, desc)
if err != nil {
return nil, err
}
Expand Down
4 changes: 2 additions & 2 deletions util/contentutil/copy_test.go
View file
Expand Up @@ -18,13 +18,13 @@ func TestCopy(t *testing.T) {
b0 := NewBuffer()
b1 := NewBuffer()

err := content.WriteBlob(ctx, b0, "foo", bytes.NewBuffer([]byte("foobar")), -1, "")
err := content.WriteBlob(ctx, b0, "foo", bytes.NewBuffer([]byte("foobar")), ocispec.Descriptor{Size: -1})
require.NoError(t, err)

err = Copy(ctx, b1, b0, ocispec.Descriptor{Digest: digest.FromBytes([]byte("foobar")), Size: -1})
require.NoError(t, err)

dt, err := content.ReadBlob(ctx, b1, digest.FromBytes([]byte("foobar")))
dt, err := content.ReadBlob(ctx, b1, ocispec.Descriptor{Digest: digest.FromBytes([]byte("foobar"))})
require.NoError(t, err)
require.Equal(t, string(dt), "foobar")
}

0 comments on commit 7943598

Please sign in to comment.