Add initial filegrain-esque continuity manifest

cmd/convert/main.go

@@ -70,7 +70,7 @@ func Convert(ctx context.Context, ipfsCln iface.CoreAPI, ctrdCln *containerd.Cli
 		return errors.Wrapf(err, "failed to create fetcher for %q", src)
 	}
 
-	converter := ipcs.NewConverter(ipfsCln, contentutil.FromFetcher(fetcher))
+	converter := ipcs.NewContinuityConverter(ipfsCln, contentutil.FromFetcher(fetcher))
 	mfstDesc, err := converter.Convert(ctx, srcDesc)
 	if err != nil {
 		return errors.Wrapf(err, "failed to convert %q to ipfs manifest", srcName)

converter.go

@@ -9,11 +9,13 @@ import (
 	"log"
 	"os"
 
+	"github.com/AkihiroSuda/filegrain/continuityutil"
 	"github.com/containerd/containerd/archive"
 	"github.com/containerd/containerd/archive/compression"
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/platforms"
+	"github.com/containerd/continuity"
 	"github.com/hinshun/ipcs/digestconv"
 	files "github.com/ipfs/go-ipfs-files"
 	iface "github.com/ipfs/interface-go-ipfs-core"
@@ -116,6 +118,7 @@ func addFile(ctx context.Context, api iface.CoreAPI, n files.Node) (digest.Diges
 	if err != nil {
 		return "", errors.Wrapf(err, "failed to convert cid %q to digest", p.Cid())
 	}
+	log.Printf("Added as %q [%s]", p.Cid(), dgst)
 
 	return dgst, nil
 }
@@ -220,3 +223,98 @@ func RegularTypeFilter(header *tar.Header) (bool, error) {
 		return false, nil
 	}
 }
+
+type continuityConverter struct {
+	api      iface.CoreAPI
+	provider content.Provider
+}
+
+func NewContinuityConverter(api iface.CoreAPI, provider content.Provider) Converter {
+	return &continuityConverter{
+		api:      api,
+		provider: provider,
+	}
+}
+
+func (c *continuityConverter) Convert(ctx context.Context, desc ocispec.Descriptor) (ocispec.Descriptor, error) {
+	mfst, err := images.Manifest(ctx, c.provider, desc, platforms.Default())
+	if err != nil {
+		return ocispec.Descriptor{}, errors.Wrap(err, "failed to get manifest")
+	}
+
+	rootfs, err := ioutil.TempDir("", "ipcs-rootfs")
+	if err != nil {
+		return ocispec.Descriptor{}, errors.Wrap(err, "failed to create tmp root directory")
+	}
+
+	for _, layer := range mfst.Layers {
+		ra, err := c.provider.ReaderAt(ctx, layer)
+		if err != nil {
+			return ocispec.Descriptor{}, err
+		}
+
+		cr := content.NewReader(ra)
+		r, err := compression.DecompressStream(cr)
+		if err != nil {
+			return ocispec.Descriptor{}, err
+		}
+
+		_, err = archive.Apply(ctx, rootfs, r, archive.WithFilter(func(hdr *tar.Header) (bool, error) {
+			return true, nil
+		}))
+		if err != nil {
+			r.Close()
+			return ocispec.Descriptor{}, err
+		}
+		r.Close()
+	}
+
+	pctx, err := continuity.NewContext(rootfs)
+	if err != nil {
+		return ocispec.Descriptor{}, errors.Wrap(err, "failed to create continuity context")
+	}
+
+	m, err := continuity.BuildManifest(pctx)
+	if err != nil {
+		return ocispec.Descriptor{}, errors.Wrap(err, "failed to build continuity manifest")
+	}
+
+	p, err := continuity.Marshal(m)
+	if err != nil {
+		return ocispec.Descriptor{}, errors.Wrap(err, "failed to marshal continuity manifest")
+	}
+
+	continuityDigest, err := addFile(ctx, c.api, files.NewBytesFile(p))
+	if err != nil {
+		return ocispec.Descriptor{}, errors.Wrap(err, "failed to upload manifest")
+	}
+	mfst.Layers = []ocispec.Descriptor{
+		{
+			MediaType: continuityutil.MediaTypeManifestV0Protobuf,
+			Digest:    continuityDigest,
+			Size:      int64(len(p)),
+		},
+	}
+
+	mfst.Config.Digest, err = copyFile(ctx, c.api, c.provider, mfst.Config)
+	if err != nil {
+		return ocispec.Descriptor{}, errors.Wrapf(err, "failed to upload manifest config blob %q", mfst.Config.Digest)
+	}
+
+	mfstJSON, err := json.MarshalIndent(mfst, "", "   ")
+	if err != nil {
+		return ocispec.Descriptor{}, errors.Wrap(err, "failed to marshal manifest JSON")
+	}
+
+	mfstDigest, err := addFile(ctx, c.api, files.NewBytesFile(mfstJSON))
+	if err != nil {
+		return ocispec.Descriptor{}, errors.Wrap(err, "failed to upload manifest")
+	}
+	log.Printf("Converted Manifest [%d] %s:\n%s", len(mfstJSON), mfstDigest, mfstJSON)
+
+	return ocispec.Descriptor{
+		MediaType: ocispec.MediaTypeImageManifest,
+		Digest:    mfstDigest,
+		Size:      int64(len(mfstJSON)),
+	}, nil
+}

go.mod

@@ -3,6 +3,7 @@ module github.com/hinshun/ipcs
 go 1.12
 
 require (
+	github.com/AkihiroSuda/filegrain v0.0.0-20190607033141-66f306520d62
 	github.com/Microsoft/go-winio v0.4.12 // indirect
 	github.com/Microsoft/hcsshim v0.8.6 // indirect
 	github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 // indirect
@@ -11,7 +12,7 @@ require (
 	github.com/containerd/cgroups v0.0.0-20190226200435-dbea6f2bd416 // indirect
 	github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50 // indirect
 	github.com/containerd/containerd v1.2.6
-	github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 // indirect
+	github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc
 	github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 // indirect
 	github.com/containerd/go-runc v0.0.0-20190226155025-7d11b49dc076 // indirect
 	github.com/containerd/ttrpc v0.0.0-20190211042230-69144327078c // indirect

go.sum

@@ -1,4 +1,6 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+github.com/AkihiroSuda/filegrain v0.0.0-20190607033141-66f306520d62 h1:r4WMZt0dA/9DIszP7ZcmBtohmgFgkEqo6dfje9upP9E=
+github.com/AkihiroSuda/filegrain v0.0.0-20190607033141-66f306520d62/go.mod h1:8sU3hhE0fWQ1+lWEVLOP9ExV/vGgMCA+Z0dsk3hD1LY=
 github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
@@ -8,6 +10,8 @@ github.com/Microsoft/go-winio v0.4.12/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcy
 github.com/Microsoft/hcsshim v0.8.6 h1:ZfF0+zZeYdzMIVMZHKtDKJvLHj76XCuVae/jNkjj0IA=
 github.com/Microsoft/hcsshim v0.8.6/go.mod h1:Op3hHsoHPAvb6lceZHDtd9OkTew38wNoXnJs8iY7rUg=
 github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/Sirupsen/logrus v0.11.5 h1:aIMrrsnipdTlAieMe7FC/iiuJ0+ELiXCT4YiVQiK9j8=
+github.com/Sirupsen/logrus v0.11.5/go.mod h1:rmk17hk6i8ZSAJkSDa7nOxamrG+SP4P0mm+DAvExv4U=
 github.com/Stebalien/go-bitfield v0.0.0-20180330043415-076a62f9ce6e/go.mod h1:3oM7gXIttpYDAJXpVNnSCiUMYBLIZ6cb1t+Ip982MRo=
 github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII=
 github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412 h1:w1UutsfOrms1J05zt7ISrnJIXKzwaspym5BTKGx93EI=
@@ -32,6 +36,8 @@ github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtE
 github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ=
+github.com/cheggaaa/pb v1.0.13 h1:vSXdStIZ5JH9knGF+zbK1gpOJZPQ+CvU24ETBe7LyS0=
+github.com/cheggaaa/pb v1.0.13/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/containerd/aufs v0.0.0-20190114185352-f894a800659b h1:MRsS/11KfevkQGq+jwFmyNKaY//l6hvFPL5/I333MAg=
 github.com/containerd/aufs v0.0.0-20190114185352-f894a800659b/go.mod h1:KWTFEKnST/R6wiSG+n7rid/ZQJxi73594So04IsWRpc=
@@ -45,6 +51,8 @@ github.com/containerd/containerd v1.2.6 h1:K38ZSAA9oKSrX3iFNY+4SddZ8hH1TCMCerc8N
 github.com/containerd/containerd v1.2.6/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA=
 github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 h1:4BX8f882bXEDKfWIf0wa8HRvpnBoPszJJXL+TVbBw4M=
 github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
+github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8=
+github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 h1:PUD50EuOMkXVcpBIA/R95d56duJR9VxhwncsFbNnxW4=
 github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI=
 github.com/containerd/go-runc v0.0.0-20190226155025-7d11b49dc076 h1:biYGul6kwz5/Lh6UxkYQyRKPeYA/ZL5n2pdCAcYUpLA=
@@ -75,6 +83,7 @@ github.com/docker/go-events v0.0.0-20170721190031-9461782956ad h1:VXIse57M5C6ezD
 github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA=
 github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 h1:X0fj836zx99zFu83v/M79DuBn84IL/Syx1SY6Y5ZEMA=
 github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI=
+github.com/docker/go-units v0.3.2/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk=
 github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
 github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
@@ -101,6 +110,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekf
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
+github.com/golang/protobuf v0.0.0-20170427213220-18c9bb326172/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.0 h1:kbxbvI4Un1LUWKxufD+BiE6AEExYYgkQLQmLFqA1LFk=
@@ -125,6 +135,8 @@ github.com/gxed/hashland/murmur3 v0.0.1 h1:SheiaIt0sda5K+8FLz952/1iWS9zrnKsEJaOJ
 github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48=
 github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824 h1:TF4mX7zXpeyz/xintezebSa7ZDxAGBnqDwcoobvaz2o=
 github.com/gxed/pubsub v0.0.0-20180201040156-26ebdf44f824/go.mod h1:OiEWyHgK+CWrmOlVquHaIK1vhpUJydC9m0Je6mhaiNE=
+github.com/hanwen/go-fuse v0.0.0-20170424203904-5404bf0e372d h1:06w1Xwq+3gQbm76tgdC8QL+2B6aBxSEMCRH73aSv+fw=
+github.com/hanwen/go-fuse v0.0.0-20170424203904-5404bf0e372d/go.mod h1:4ZJ05v9yt5k/mcFkGvSPKJB5T8G/6nuumL63ZqlrPvI=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -368,6 +380,8 @@ github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0X
 github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ=
 github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw=
 github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s=
+github.com/mattn/go-runewidth v0.0.2 h1:UnlwIPBGaTZfPQ6T1IGzPI0EkYAQmT9fAEJ/poFC63o=
+github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU=
 github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
 github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
@@ -419,8 +433,10 @@ github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
+github.com/opencontainers/go-digest v1.0.0-rc0/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
+github.com/opencontainers/image-spec v0.0.0-20170501194034-c87455c1b399/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI=
 github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0=
 github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y=
@@ -467,6 +483,12 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72 h1:qLC7fQah7D6K1
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI=
 github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/spf13/cobra v0.0.0-20170501210834-69f86e6d5d7a h1:9OzjTlfCOEYy/+ivO76kJxif+t9vmOjlXZDL0cnxdvg=
+github.com/spf13/cobra v0.0.0-20170501210834-69f86e6d5d7a/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ=
+github.com/spf13/pflag v0.0.0-20170427125145-f1d95a35e132 h1:Z9FpVnwd+lo1kp/Mf5kL1XW57L+Vcu/40wC3/aFGCHY=
+github.com/spf13/pflag v0.0.0-20170427125145-f1d95a35e132/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
+github.com/stevvooe/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:1GLiICsIP1hnDnXS8MFG17DyorbYXMTFipmAfAOrTKk=
+github.com/stevvooe/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:AXZEju8Nky8hvQW5KS9REMxcmvWpBFJRqnhQ6W/7S6E=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
@@ -544,6 +566,8 @@ golang.org/x/sys v0.0.0-20190302025703-b6889370fb10 h1:xQJI9OEiErEQ++DoXOHqEpzsG
 golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d h1:+R4KGOnez64A81RvjARKc4UT5/tI9ujCIVX+P5KiHuI=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444 h1:/d2cWp6PSamH4jDPFLyO150psQdqvtoNX8Zjg3AQ31g=
+golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=