Enable to distribute images on IPFS

Signed-off-by: Kohei Tokunaga <ktokunaga.mail@gmail.com>

.github/workflows/test.yml

@@ -73,7 +73,7 @@ jobs:
     - name: "Register QEMU (tonistiigi/binfmt)"
       run: docker run --privileged --rm tonistiigi/binfmt --install all
     - name: "Run integration tests"
-      run: docker run -t --rm --privileged test-integration go test -v ./cmd/nerdctl/... -args -test.kill-daemon
+      run: docker run -t --rm --privileged test-integration /test-integration.sh go test -v ./cmd/nerdctl/... -args -test.kill-daemon
 
   test-integration-rootless:
     runs-on: ubuntu-20.04

Dockerfile

@@ -213,7 +213,8 @@ COPY . /go/src/github.com/containerd/nerdctl
 WORKDIR /go/src/github.com/containerd/nerdctl
 VOLUME /tmp
 ENV CGO_ENABLED=0
-CMD ["go", "test", "-v", "./cmd/nerdctl/..."]
+COPY ./Dockerfile.d/test-integration.sh /
+CMD ["/test-integration.sh", "go", "test", "-v", "./cmd/nerdctl/..."]
 
 FROM test-integration AS test-integration-rootless
 # Install SSH for creating systemd user session.

Dockerfile.d/test-integration-rootless.sh

@@ -21,5 +21,26 @@ if [[ "$(id -u)" = "0" ]]; then
 else
 	containerd-rootless-setuptool.sh install
 	containerd-rootless-setuptool.sh install-buildkit
+        containerd-rootless-setuptool.sh install-stargz
+        cat <<EOF > /home/rootless/.config/containerd/config.toml
+[proxy_plugins]
+  [proxy_plugins."stargz"]
+    type = "snapshot"
+    address = "/run/user/1000/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+EOF
+        systemctl --user restart containerd.service
+        sleep 3
+        if [ -d "${HOME}/.ipfs" ] ; then
+            echo "IPFS has already been set up."
+        else
+            export IPFS_PATH=${HOME}/.ipfs
+            mkdir -p ${IPFS_PATH}
+            nerdctl run -d --name ipfs_host -v ${IPFS_PATH}:/data/ipfs -p 127.0.0.1:5001:5001 \
+                    ghcr.io/stargz-containers/ipfs/go-ipfs:v0.10.0-org  daemon --offline
+            sleep 3
+            echo "ipfs = true" > ${HOME}/.config/containerd-stargz-grpc/config.toml
+            systemctl --user restart stargz-snapshotter.service
+            sleep 3
+        fi
 	exec "$@"
 fi

Dockerfile.d/test-integration.sh

@@ -0,0 +1,32 @@
+#!/bin/bash
+
+#   Copyright The containerd Authors.
+
+#   Licensed under the Apache License, Version 2.0 (the "License");
+#   you may not use this file except in compliance with the License.
+#   You may obtain a copy of the License at
+
+#       http://www.apache.org/licenses/LICENSE-2.0
+
+#   Unless required by applicable law or agreed to in writing, software
+#   distributed under the License is distributed on an "AS IS" BASIS,
+#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#   See the License for the specific language governing permissions and
+#   limitations under the License.
+
+set -eux -o pipefail
+if [ -d "${HOME}/.ipfs" ] ; then
+    echo "IPFS has already been set up."
+else
+    sleep 3 # wait for containerd starting
+    export IPFS_PATH=${HOME}/.ipfs
+    mkdir -p ${IPFS_PATH}
+    nerdctl run -d --name ipfs_host -v ${IPFS_PATH}:/data/ipfs -p 127.0.0.1:5001:5001 \
+            ghcr.io/stargz-containers/ipfs/go-ipfs:v0.10.0-org  daemon --offline
+    sleep 3
+    mkdir -p /etc/containerd-stargz-grpc
+    echo "ipfs = true" > /etc/containerd-stargz-grpc/config.toml
+    systemctl restart stargz-snapshotter.service
+    sleep 3
+fi
+exec "$@"

cmd/nerdctl/commit.go

@@ -21,9 +21,9 @@ import (
 	"errors"
 	"fmt"
 
-	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/nerdctl/pkg/idutil/containerwalker"
 	"github.com/containerd/nerdctl/pkg/imgutil/commit"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
 
 	"github.com/spf13/cobra"
 )
@@ -85,7 +85,7 @@ func commitAction(cmd *cobra.Command, args []string) error {
 func newCommitOpts(cmd *cobra.Command, args []string) (*commit.Opts, error) {
 	rawRef := args[1]
 
-	named, err := refdocker.ParseDockerRef(rawRef)
+	named, err := referenceutil.ParseDockerRef(rawRef)
 	if err != nil {
 		return nil, err
 	}

cmd/nerdctl/compose.go

@@ -24,10 +24,12 @@ import (
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/platforms"
-	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/nerdctl/pkg/composer"
 	"github.com/containerd/nerdctl/pkg/imgutil"
+	"github.com/containerd/nerdctl/pkg/ipfs"
 	"github.com/containerd/nerdctl/pkg/netutil"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
+	httpapi "github.com/ipfs/go-ipfs-http-client"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 
 	"github.com/spf13/cobra"
@@ -145,11 +147,11 @@ func getComposer(cmd *cobra.Command, client *containerd.Client) (*composer.Compo
 	}
 
 	o.ImageExists = func(ctx context.Context, rawRef string) (bool, error) {
-		named, err := refdocker.ParseDockerRef(rawRef)
+		refNamed, err := referenceutil.ParseAny(rawRef)
 		if err != nil {
 			return false, err
 		}
-		ref := named.String()
+		ref := refNamed.String()
 		if _, err := client.ImageService().Get(ctx, ref); err != nil {
 			if errors.Is(err, errdefs.ErrNotFound) {
 				return false, nil
@@ -168,8 +170,18 @@ func getComposer(cmd *cobra.Command, client *containerd.Client) (*composer.Compo
 			}
 			ocispecPlatforms = []ocispec.Platform{parsed} // no append
 		}
-		_, imgErr := imgutil.EnsureImage(ctx, client, cmd.OutOrStdout(), snapshotter, imageName,
-			pullMode, insecure, ocispecPlatforms, nil)
+		var imgErr error
+		if scheme, ref, err := referenceutil.ParseIPFSRefWithScheme(imageName); err == nil {
+			ipfsClient, err := httpapi.NewLocalApi()
+			if err != nil {
+				return err
+			}
+			_, imgErr = ipfs.EnsureImage(ctx, client, ipfsClient, cmd.OutOrStdout(), snapshotter, scheme, ref,
+				pullMode, ocispecPlatforms, nil)
+		} else {
+			_, imgErr = imgutil.EnsureImage(ctx, client, cmd.OutOrStdout(), snapshotter, imageName,
+				pullMode, insecure, ocispecPlatforms, nil)
+		}
 		return imgErr
 	}
 

cmd/nerdctl/compose_up_test.go

@@ -19,6 +19,7 @@ package main
 import (
 	"fmt"
 	"io"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -30,8 +31,51 @@ import (
 
 func TestComposeUp(t *testing.T) {
 	base := testutil.NewBase(t)
+	testComposeUp(t, base, fmt.Sprintf(`
+version: '3.1'
 
-	var dockerComposeYAML = fmt.Sprintf(`
+services:
+
+  wordpress:
+    image: %s
+    restart: always
+    ports:
+      - 8080:80
+    environment:
+      WORDPRESS_DB_HOST: db
+      WORDPRESS_DB_USER: exampleuser
+      WORDPRESS_DB_PASSWORD: examplepass
+      WORDPRESS_DB_NAME: exampledb
+    volumes:
+      - wordpress:/var/www/html
+
+  db:
+    image: %s
+    restart: always
+    environment:
+      MYSQL_DATABASE: exampledb
+      MYSQL_USER: exampleuser
+      MYSQL_PASSWORD: examplepass
+      MYSQL_RANDOM_ROOT_PASSWORD: '1'
+    volumes:
+      - db:/var/lib/mysql
+
+volumes:
+  wordpress:
+  db:
+`, testutil.WordpressImage, testutil.MariaDBImage))
+}
+
+func TestIPFSComposeUp(t *testing.T) {
+	testutil.DockerIncompatible(t)
+	base := testutil.NewBase(t)
+	requiresStargz(base)
+	base.Env = append(os.Environ(), "CONTAINERD_SNAPSHOTTER=stargz")
+	ipfsImgs := make([]string, 2)
+	for i, img := range []string{testutil.WordpressImage, testutil.MariaDBImage} {
+		ipfsImgs[i] = pushImageToIPFS(t, base, img)
+	}
+	testComposeUp(t, base, fmt.Sprintf(`
 version: '3.1'
 
 services:
@@ -47,6 +91,8 @@ services:
       WORDPRESS_DB_PASSWORD: examplepass
       WORDPRESS_DB_NAME: exampledb
     volumes:
+      # workaround for https://github.com/containerd/stargz-snapshotter/issues/444
+      - "/run"
       - wordpress:/var/www/html
 
   db:
@@ -58,12 +104,17 @@ services:
       MYSQL_PASSWORD: examplepass
       MYSQL_RANDOM_ROOT_PASSWORD: '1'
     volumes:
+      # workaround for https://github.com/containerd/stargz-snapshotter/issues/444
+      - "/run"
       - db:/var/lib/mysql
 
 volumes:
   wordpress:
   db:
-`, testutil.WordpressImage, testutil.MariaDBImage)
+`, ipfsImgs[0], ipfsImgs[1]))
+}
+
+func testComposeUp(t *testing.T, base *testutil.Base, dockerComposeYAML string) {
 	comp := testutil.NewComposeDir(t, dockerComposeYAML)
 	defer comp.CleanUp()
 

cmd/nerdctl/image_convert.go

@@ -25,8 +25,8 @@ import (
 
 	"github.com/containerd/containerd/images/converter"
 	"github.com/containerd/containerd/images/converter/uncompress"
-	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/nerdctl/pkg/platformutil"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
 	"github.com/containerd/stargz-snapshotter/estargz"
 	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
 	"github.com/containerd/stargz-snapshotter/recorder"
@@ -90,13 +90,13 @@ func imageConvertAction(cmd *cobra.Command, args []string) error {
 		return errors.New("src and target image need to be specified")
 	}
 
-	srcNamed, err := refdocker.ParseDockerRef(srcRawRef)
+	srcNamed, err := referenceutil.ParseAny(srcRawRef)
 	if err != nil {
 		return err
 	}
 	srcRef := srcNamed.String()
 
-	targetNamed, err := refdocker.ParseDockerRef(targetRawRef)
+	targetNamed, err := referenceutil.ParseDockerRef(targetRawRef)
 	if err != nil {
 		return err
 	}

cmd/nerdctl/image_cryptutil.go

@@ -23,10 +23,10 @@ import (
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images/converter"
-	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/imgcrypt/images/encryption"
 	"github.com/containerd/imgcrypt/images/encryption/parsehelpers"
 	"github.com/containerd/nerdctl/pkg/platformutil"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/spf13/cobra"
 )
@@ -105,13 +105,13 @@ func getImgcryptAction(encrypt bool) func(cmd *cobra.Command, args []string) err
 			return errors.New("src and target image need to be specified")
 		}
 
-		srcNamed, err := refdocker.ParseDockerRef(srcRawRef)
+		srcNamed, err := referenceutil.ParseAny(srcRawRef)
 		if err != nil {
 			return err
 		}
 		srcRef := srcNamed.String()
 
-		targetNamed, err := refdocker.ParseDockerRef(targetRawRef)
+		targetNamed, err := referenceutil.ParseDockerRef(targetRawRef)
 		if err != nil {
 			return err
 		}

cmd/nerdctl/images.go

@@ -34,10 +34,10 @@ import (
 	"github.com/containerd/containerd/images"
 	"github.com/containerd/containerd/pkg/progress"
 	"github.com/containerd/containerd/platforms"
-	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/containerd/snapshots"
 	"github.com/containerd/nerdctl/pkg/formatter"
 	"github.com/containerd/nerdctl/pkg/imgutil"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
 	"github.com/opencontainers/image-spec/identity"
 	v1 "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/sirupsen/logrus"
@@ -79,7 +79,7 @@ func imagesAction(cmd *cobra.Command, args []string) error {
 	}
 
 	if len(args) > 0 {
-		canonicalRef, err := refdocker.ParseDockerRef(args[0])
+		canonicalRef, err := referenceutil.ParseAny(args[0])
 		if err != nil {
 			return err
 		}

cmd/nerdctl/pull.go

@@ -21,16 +21,19 @@ import (
 	"os"
 
 	"github.com/containerd/nerdctl/pkg/imgutil"
+	"github.com/containerd/nerdctl/pkg/ipfs"
 	"github.com/containerd/nerdctl/pkg/platformutil"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
 	"github.com/containerd/nerdctl/pkg/strutil"
+	httpapi "github.com/ipfs/go-ipfs-http-client"
 
 	"github.com/spf13/cobra"
 )
 
 func newPullCommand() *cobra.Command {
 	var pullCommand = &cobra.Command{
 		Use:           "pull",
-		Short:         "Pull an image from a registry",
+		Short:         "Pull an image from a registry. Optionally specify \"ipfs://\" or \"ipns://\" scheme to pull image from IPFS.",
 		RunE:          pullAction,
 		SilenceUsage:  true,
 		SilenceErrors: true,
@@ -88,6 +91,17 @@ func pullAction(cmd *cobra.Command, args []string) error {
 	if err != nil {
 		return err
 	}
+
+	if scheme, ref, err := referenceutil.ParseIPFSRefWithScheme(args[0]); err == nil {
+		ipfsClient, err := httpapi.NewLocalApi()
+		if err != nil {
+			return err
+		}
+		_, err = ipfs.EnsureImage(ctx, client, ipfsClient, os.Stdout, snapshotter, scheme, ref,
+			"always", ocispecPlatforms, unpack)
+		return err
+	}
+
 	_, err = imgutil.EnsureImage(ctx, client, os.Stdout, snapshotter, args[0],
 		"always", insecure, ocispecPlatforms, unpack)
 	return err