Enable to distribute images on IPFS

Signed-off-by: Kohei Tokunaga <ktokunaga.mail@gmail.com>

cmd/nerdctl/commit.go

@@ -21,9 +21,9 @@ import (
 	"errors"
 	"fmt"
 
-	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/nerdctl/pkg/idutil/containerwalker"
 	"github.com/containerd/nerdctl/pkg/imgutil/commit"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
 
 	"github.com/spf13/cobra"
 )
@@ -85,7 +85,7 @@ func commitAction(cmd *cobra.Command, args []string) error {
 func newCommitOpts(cmd *cobra.Command, args []string) (*commit.Opts, error) {
 	rawRef := args[1]
 
-	named, err := refdocker.ParseDockerRef(rawRef)
+	named, err := referenceutil.ParseDockerRef(rawRef)
 	if err != nil {
 		return nil, err
 	}

cmd/nerdctl/compose.go

@@ -24,10 +24,12 @@ import (
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/platforms"
-	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/nerdctl/pkg/composer"
 	"github.com/containerd/nerdctl/pkg/imgutil"
+	"github.com/containerd/nerdctl/pkg/ipfs"
 	"github.com/containerd/nerdctl/pkg/netutil"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
+	httpapi "github.com/ipfs/go-ipfs-http-client"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 
 	"github.com/spf13/cobra"
@@ -145,11 +147,11 @@ func getComposer(cmd *cobra.Command, client *containerd.Client) (*composer.Compo
 	}
 
 	o.ImageExists = func(ctx context.Context, rawRef string) (bool, error) {
-		named, err := refdocker.ParseDockerRef(rawRef)
+		refNamed, err := referenceutil.ParseAny(rawRef)
 		if err != nil {
 			return false, err
 		}
-		ref := named.String()
+		ref := refNamed.String()
 		if _, err := client.ImageService().Get(ctx, ref); err != nil {
 			if errors.Is(err, errdefs.ErrNotFound) {
 				return false, nil
@@ -168,8 +170,18 @@ func getComposer(cmd *cobra.Command, client *containerd.Client) (*composer.Compo
 			}
 			ocispecPlatforms = []ocispec.Platform{parsed} // no append
 		}
-		_, imgErr := imgutil.EnsureImage(ctx, client, cmd.OutOrStdout(), snapshotter, imageName,
-			pullMode, insecure, ocispecPlatforms, nil)
+		var imgErr error
+		if scheme, ref, err := referenceutil.ParseIPFSRefWithScheme(imageName); err == nil {
+			ipfsClient, err := httpapi.NewLocalApi()
+			if err != nil {
+				return err
+			}
+			_, imgErr = ipfs.EnsureImage(ctx, client, ipfsClient, cmd.OutOrStdout(), snapshotter, scheme, ref,
+				pullMode, ocispecPlatforms, nil)
+		} else {
+			_, imgErr = imgutil.EnsureImage(ctx, client, cmd.OutOrStdout(), snapshotter, imageName,
+				pullMode, insecure, ocispecPlatforms, nil)
+		}
 		return imgErr
 	}
 

cmd/nerdctl/image_convert.go

@@ -25,8 +25,8 @@ import (
 
 	"github.com/containerd/containerd/images/converter"
 	"github.com/containerd/containerd/images/converter/uncompress"
-	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/nerdctl/pkg/platformutil"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
 	"github.com/containerd/stargz-snapshotter/estargz"
 	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
 	"github.com/containerd/stargz-snapshotter/recorder"
@@ -90,13 +90,13 @@ func imageConvertAction(cmd *cobra.Command, args []string) error {
 		return errors.New("src and target image need to be specified")
 	}
 
-	srcNamed, err := refdocker.ParseDockerRef(srcRawRef)
+	srcNamed, err := referenceutil.ParseAny(srcRawRef)
 	if err != nil {
 		return err
 	}
 	srcRef := srcNamed.String()
 
-	targetNamed, err := refdocker.ParseDockerRef(targetRawRef)
+	targetNamed, err := referenceutil.ParseDockerRef(targetRawRef)
 	if err != nil {
 		return err
 	}

cmd/nerdctl/image_cryptutil.go

@@ -23,10 +23,10 @@ import (
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images/converter"
-	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/imgcrypt/images/encryption"
 	"github.com/containerd/imgcrypt/images/encryption/parsehelpers"
 	"github.com/containerd/nerdctl/pkg/platformutil"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/spf13/cobra"
 )
@@ -105,13 +105,13 @@ func getImgcryptAction(encrypt bool) func(cmd *cobra.Command, args []string) err
 			return errors.New("src and target image need to be specified")
 		}
 
-		srcNamed, err := refdocker.ParseDockerRef(srcRawRef)
+		srcNamed, err := referenceutil.ParseAny(srcRawRef)
 		if err != nil {
 			return err
 		}
 		srcRef := srcNamed.String()
 
-		targetNamed, err := refdocker.ParseDockerRef(targetRawRef)
+		targetNamed, err := referenceutil.ParseDockerRef(targetRawRef)
 		if err != nil {
 			return err
 		}

cmd/nerdctl/images.go

@@ -34,10 +34,10 @@ import (
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/pkg/progress"
 	"github.com/containerd/containerd/platforms"
-	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/containerd/snapshots"
 	"github.com/containerd/nerdctl/pkg/formatter"
 	"github.com/containerd/nerdctl/pkg/imgutil"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
 	"github.com/opencontainers/image-spec/identity"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/sirupsen/logrus"
@@ -79,7 +79,7 @@ func imagesAction(cmd *cobra.Command, args []string) error {
 	}
 
 	if len(args) > 0 {
-		canonicalRef, err := refdocker.ParseDockerRef(args[0])
+		canonicalRef, err := referenceutil.ParseAny(args[0])
 		if err != nil {
 			return err
 		}

cmd/nerdctl/pull.go

@@ -21,16 +21,19 @@ import (
 	"os"
 
 	"github.com/containerd/nerdctl/pkg/imgutil"
+	"github.com/containerd/nerdctl/pkg/ipfs"
 	"github.com/containerd/nerdctl/pkg/platformutil"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
 	"github.com/containerd/nerdctl/pkg/strutil"
+	httpapi "github.com/ipfs/go-ipfs-http-client"
 
 	"github.com/spf13/cobra"
 )
 
 func newPullCommand() *cobra.Command {
 	var pullCommand = &cobra.Command{
 		Use:           "pull",
-		Short:         "Pull an image from a registry",
+		Short:         "Pull an image from a registry. Optionally specify \"ipfs://\" or \"ipns://\" scheme to pull image from IPFS.",
 		RunE:          pullAction,
 		SilenceUsage:  true,
 		SilenceErrors: true,
@@ -88,6 +91,17 @@ func pullAction(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
+
+	if scheme, ref, err := referenceutil.ParseIPFSRefWithScheme(args[0]); err == nil {
+		ipfsClient, err := httpapi.NewLocalApi()
+		if err != nil {
+			return err
+		}
+		_, err = ipfs.EnsureImage(ctx, client, ipfsClient, os.Stdout, snapshotter, scheme, ref,
+			"always", ocispecPlatforms, unpack)
+		return err
+	}
+
 	_, err = imgutil.EnsureImage(ctx, client, os.Stdout, snapshotter, args[0],
 		"always", insecure, ocispecPlatforms, unpack)
 	return err

cmd/nerdctl/push.go

@@ -17,24 +17,35 @@
 package main
 
 import (
+	"context"
 	"errors"
 	"fmt"
+	"io"
 
+	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images/converter"
 	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/containerd/remotes"
 	"github.com/containerd/nerdctl/pkg/imgutil"
 	"github.com/containerd/nerdctl/pkg/imgutil/dockerconfigresolver"
 	"github.com/containerd/nerdctl/pkg/imgutil/push"
+	"github.com/containerd/nerdctl/pkg/ipfs"
 	"github.com/containerd/nerdctl/pkg/platformutil"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
+	"github.com/containerd/stargz-snapshotter/estargz"
+	"github.com/containerd/stargz-snapshotter/estargz/zstdchunked"
+	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
+	httpapi "github.com/ipfs/go-ipfs-http-client"
+	digest "github.com/opencontainers/go-digest"
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 )
 
 func newPushCommand() *cobra.Command {
 	var pushCommand = &cobra.Command{
 		Use:               "push NAME[:TAG]",
-		Short:             "Push an image or a repository to a registry",
+		Short:             "Push an image or a repository to a registry. Optionally specify \"ipfs://\" or \"ipns://\" scheme to push image to IPFS.",
 		RunE:              pushAction,
 		ValidArgsFunction: pushShellComplete,
 		SilenceUsage:      true,
@@ -47,6 +58,11 @@ func newPushCommand() *cobra.Command {
 	pushCommand.Flags().Bool("all-platforms", false, "Push content for all platforms")
 	// #endregion
 
+	pushCommand.PersistentFlags().Bool("estargz", false, "Convert the image into eStargz (always true in IPFS mode)")
+
+	pushCommand.PersistentFlags().Bool("ipfs-no-estargz", false, "Disable conversion of the image into eStargz in IPFS mode")
+	pushCommand.PersistentFlags().Bool("ipfs-ensure-image", true, "Ensure the IPFS image is fully pulled to containerd")
+
 	return pushCommand
 }
 
@@ -55,53 +71,99 @@ func pushAction(cmd *cobra.Command, args []string) error {
 		return errors.New("image name needs to be specified")
 	}
 	rawRef := args[0]
-	named, err := refdocker.ParseDockerRef(rawRef)
+
+	client, ctx, cancel, err := newClient(cmd)
 	if err != nil {
 		return err
 	}
-	ref := named.String()
-	refDomain := refdocker.Domain(named)
+	defer cancel()
 
-	insecure, err := cmd.Flags().GetBool("insecure-registry")
+	allPlatforms, err := cmd.Flags().GetBool("all-platforms")
 	if err != nil {
 		return err
 	}
-	client, ctx, cancel, err := newClient(cmd)
+	platform, err := cmd.Flags().GetStringSlice("platform")
 	if err != nil {
 		return err
 	}
-	defer cancel()
 
-	allPlatforms, err := cmd.Flags().GetBool("all-platforms")
+	if scheme, ref, err := referenceutil.ParseIPFSRefWithScheme(rawRef); err == nil {
+		if scheme != "ipfs" {
+			return fmt.Errorf("ipfs scheme is only supported but got %q", scheme)
+		}
+		noConversion, err := cmd.Flags().GetBool("ipfs-no-estargz")
+		if err != nil {
+			return err
+		}
+		ensureImage, err := cmd.Flags().GetBool("ipfs-ensure-image")
+		if err != nil {
+			return err
+		}
+		logrus.Infof("pushing image %q to IPFS", ref)
+		ipfsClient, err := httpapi.NewLocalApi()
+		if err != nil {
+			return err
+		}
+		var layerConvert converter.ConvertFunc
+		if !noConversion {
+			layerConvert = eStargzConvertFunc()
+		}
+		c, err := ipfs.Push(ctx, client, ipfsClient, ref, layerConvert, allPlatforms, platform, ensureImage)
+		if err != nil {
+			return err
+		}
+		fmt.Fprintln(cmd.OutOrStdout(), c.String())
+		return err
+	}
+
+	named, err := refdocker.ParseDockerRef(rawRef)
 	if err != nil {
 		return err
 	}
-	platform, err := cmd.Flags().GetStringSlice("platform")
+	ref := named.String()
+	refDomain := refdocker.Domain(named)
+
+	insecure, err := cmd.Flags().GetBool("insecure-registry")
 	if err != nil {
 		return err
 	}
+
 	platMC, err := platformutil.NewMatchComparer(allPlatforms, platform)
 	if err != nil {
 		return err
 	}
-	platRef := ref
+	pushRef := ref
 	if !allPlatforms {
-		platRef = ref + "-tmp-reduced-platform"
+		pushRef = ref + "-tmp-reduced-platform"
 		// Push fails with "400 Bad Request" when the manifest is multi-platform but we do not locally have multi-platform blobs.
 		// So we create a tmp reduced-platform image to avoid the error.
-		platImg, err := converter.Convert(ctx, client, platRef, ref, converter.WithPlatform(platMC))
+		platImg, err := converter.Convert(ctx, client, pushRef, ref, converter.WithPlatform(platMC))
 		if err != nil {
 			if len(platform) == 0 {
-				return fmt.Errorf("failed to create a tmp single-platform image %q: %w", platRef, err)
+				return fmt.Errorf("failed to create a tmp single-platform image %q: %w", pushRef, err)
 			}
-			return fmt.Errorf("failed to create a tmp reduced-platform image %q (platform=%v): %w", platRef, platform, err)
+			return fmt.Errorf("failed to create a tmp reduced-platform image %q (platform=%v): %w", pushRef, platform, err)
 		}
 		defer client.ImageService().Delete(ctx, platImg.Name)
 		logrus.Infof("pushing as a reduced-platform image (%s, %s)", platImg.Target.MediaType, platImg.Target.Digest)
 	}
 
+	convertEStargz, err := cmd.Flags().GetBool("estargz")
+	if err != nil {
+		return err
+	}
+	if convertEStargz {
+		pushRef = ref + "-tmp-esgz"
+		esgzImg, err := converter.Convert(ctx, client, pushRef, ref, converter.WithPlatform(platMC), converter.WithLayerConvertFunc(eStargzConvertFunc()))
+		if err != nil {
+			return fmt.Errorf("failed to convert to eStargz: %v", err)
+		}
+		defer client.ImageService().Delete(ctx, esgzImg.Name)
+		logrus.Infof("pushing as an eStargz image (%s, %s)", esgzImg.Target.MediaType, esgzImg.Target.Digest)
+	}
+
 	pushFunc := func(r remotes.Resolver) error {
-		return push.Push(ctx, client, r, cmd.OutOrStdout(), platRef, ref, platMC)
+		return push.Push(ctx, client, r, cmd.OutOrStdout(), pushRef, ref, platMC)
 	}
 
 	var dOpts []dockerconfigresolver.Opt
@@ -138,3 +200,44 @@ func pushShellComplete(cmd *cobra.Command, args []string, toComplete string) ([]
 	// show image names
 	return shellCompleteImageNames(cmd)
 }
+
+func eStargzConvertFunc() converter.ConvertFunc {
+	convertToESGZ := estargzconvert.LayerConvertFunc()
+	return func(ctx context.Context, cs content.Store, desc ocispec.Descriptor) (*ocispec.Descriptor, error) {
+		if isReusableESGZ(ctx, cs, desc) {
+			logrus.Infof("reusing estargz %s without conversion", desc.Digest)
+			return nil, nil
+		}
+		newDesc, err := convertToESGZ(ctx, cs, desc)
+		if err != nil {
+			return nil, err
+		}
+		logrus.Infof("converted %q to %s", desc.MediaType, newDesc.Digest)
+		return newDesc, err
+	}
+
+}
+
+func isReusableESGZ(ctx context.Context, cs content.Store, desc ocispec.Descriptor) bool {
+	dgstStr, ok := desc.Annotations[estargz.TOCJSONDigestAnnotation]
+	if !ok {
+		return false
+	}
+	tocdgst, err := digest.Parse(dgstStr)
+	if err != nil {
+		return false
+	}
+	ra, err := cs.ReaderAt(ctx, desc)
+	if err != nil {
+		return false
+	}
+	defer ra.Close()
+	r, err := estargz.Open(io.NewSectionReader(ra, 0, desc.Size), estargz.WithDecompressors(new(zstdchunked.Decompressor)))
+	if err != nil {
+		return false
+	}
+	if _, err := r.VerifyTOC(tocdgst); err != nil {
+		return false
+	}
+	return true
+}