Enable image distribution on IPFS

.github/workflows/test.yml

@@ -52,7 +52,7 @@ jobs:
 
   test-integration:
     runs-on: ubuntu-20.04
-    timeout-minutes: 20
+    timeout-minutes: 30
     strategy:
       matrix:
         containerd: [1.4.5, 1.5.7, 1.6.0-beta.2]
@@ -77,7 +77,7 @@ jobs:
 
   test-integration-rootless:
     runs-on: ubuntu-20.04
-    timeout-minutes: 20
+    timeout-minutes: 30
     strategy:
       matrix:
         containerd: [1.4.5, 1.5.7, 1.6.0-beta.2]
@@ -100,7 +100,7 @@ jobs:
 
   cross:
     runs-on: ubuntu-20.04
-    timeout-minutes: 20
+    timeout-minutes: 30
     steps:
     - uses: actions/setup-go@v2
       with:
@@ -129,7 +129,7 @@ jobs:
   test-integration-cgroup2:
     # nested virtualization is only available on macOS hosts
     runs-on: macos-10.15
-    timeout-minutes: 40
+    timeout-minutes: 50
     env:
       VAGRANT_VAGRANTFILE: hack/Vagrantfile.fedora
     steps:

Dockerfile

@@ -34,6 +34,8 @@ ARG SLIRP4NETNS_VERSION=1.1.12
 # Extra deps: FUSE-OverlayFS
 ARG FUSE_OVERLAYFS_VERSION=1.7.1
 ARG CONTAINERD_FUSE_OVERLAYFS_VERSION=1.0.3
+# Extra deps: IPFS
+ARG IPFS_VERSION=0.10.0
 
 # Test deps
 ARG GO_VERSION=1.17
@@ -161,6 +163,15 @@ RUN fname="containerd-fuse-overlayfs-${CONTAINERD_FUSE_OVERLAYFS_VERSION}-${TARG
   tar xzf "${fname}" -C /out/bin && \
   rm -f "${fname}" && \
   echo "- containerd-fuse-overlayfs: v${CONTAINERD_FUSE_OVERLAYFS_VERSION}" >> /out/share/doc/nerdctl-full/README.md
+ARG IPFS_VERSION
+RUN fname="go-ipfs_v${IPFS_VERSION}_${TARGETOS:-linux}-${TARGETARCH:-amd64}.tar.gz" && \
+  curl -o "${fname}" -fSL "https://github.com/ipfs/go-ipfs/releases/download/v${IPFS_VERSION}/${fname}" && \
+  grep "${fname}" "/SHA256SUMS.d/go-ipfs-${IPFS_VERSION}" | sha512sum -c && \
+  tmpout=$(mktemp -d) && \
+  tar -C ${tmpout} -xzf "${fname}" go-ipfs/ipfs && \
+  mv ${tmpout}/go-ipfs/ipfs /out/bin/ && \
+  echo "- IPFS: v${IPFS_VERSION}" >> /out/share/doc/nerdctl-full/README.md
+
 RUN echo "" >> /out/share/doc/nerdctl-full/README.md && \
   echo "## License" >> /out/share/doc/nerdctl-full/README.md && \
   echo "- bin/slirp4netns:    [GNU GENERAL PUBLIC LICENSE, Version 2](https://github.com/rootless-containers/slirp4netns/blob/v${SLIRP4NETNS_VERSION}/COPYING)" >> /out/share/doc/nerdctl-full/README.md && \
@@ -213,6 +224,14 @@ COPY . /go/src/github.com/containerd/nerdctl
 WORKDIR /go/src/github.com/containerd/nerdctl
 VOLUME /tmp
 ENV CGO_ENABLED=0
+# enable offline ipfs for integration test
+COPY ./Dockerfile.d/test-integration-etc_containerd-stargz-grpc_config.toml /etc/containerd-stargz-grpc/config.toml
+COPY ./Dockerfile.d/test-integration-ipfs-offline.service /usr/local/lib/systemd/system/
+# install ipfs service. avoid using 5001(api)/8080(gateway) which are reserved by tests.
+RUN systemctl enable test-integration-ipfs-offline && \
+    ipfs init && \
+    ipfs config Addresses.API "/ip4/127.0.0.1/tcp/5888" && \
+    ipfs config Addresses.Gateway "/ip4/127.0.0.1/tcp/5889"
 CMD ["go", "test", "-v", "./cmd/nerdctl/..."]
 
 FROM test-integration AS test-integration-rootless
@@ -231,6 +250,8 @@ RUN ssh-keygen -q -t rsa -f /root/.ssh/id_rsa -N '' && \
   cp -a /root/.ssh/id_rsa.pub /home/rootless/.ssh/authorized_keys && \
   mkdir -p /home/rootless/.local/share && \
   chown -R rootless:rootless /home/rootless
+# ipfs daemon for rootless containerd will be enabled in /test-integration-rootless.sh
+RUN systemctl disable test-integration-ipfs-offline
 VOLUME /home/rootless/.local/share
 RUN go test -o /usr/local/bin/nerdctl.test -c ./cmd/nerdctl
 COPY ./Dockerfile.d/test-integration-rootless.sh /

Dockerfile.d/SHA256SUMS.d/go-ipfs-0.10.0

@@ -0,0 +1,3 @@
+# From https://github.com/ipfs/go-ipfs/releases
+519d912be9367b8d5de75866aab1989954018ee1d414733ce0283735f806decb4fd11cffcb42f1beb638348a05cae95e3271c5121132e9e8e6b91ad0bfb1fda4  go-ipfs_v0.10.0_linux-amd64.tar.gz
+82a9b7a3e8701b982daa9ffbb9565ee725b50dc367a4358ab2a67e97a3b8b29b48d6c607e7edb54608181171dd9b4eded5b88bc4018b4dc348bac64de0edd26a  go-ipfs_v0.10.0_linux-arm64.tar.gz

Dockerfile.d/test-integration-etc_containerd-stargz-grpc_config.toml

@@ -0,0 +1,2 @@
+# Enable IPFS
+ipfs = true

Dockerfile.d/test-integration-ipfs-offline.service

@@ -0,0 +1,8 @@
+[Unit]
+Description=ipfs daemon for integration test (offline)
+
+[Service]
+ExecStart=ipfs daemon --init --offline
+
+[Install]
+WantedBy=docker-entrypoint.target

Dockerfile.d/test-integration-rootless.sh

@@ -21,5 +21,17 @@ if [[ "$(id -u)" = "0" ]]; then
 else
 	containerd-rootless-setuptool.sh install
 	containerd-rootless-setuptool.sh install-buildkit
+        containerd-rootless-setuptool.sh install-stargz
+        cat <<EOF >> /home/rootless/.config/containerd/config.toml
+[proxy_plugins]
+  [proxy_plugins."stargz"]
+    type = "snapshot"
+    address = "/run/user/1000/containerd-stargz-grpc/containerd-stargz-grpc.sock"
+EOF
+        systemctl --user restart containerd.service
+        containerd-rootless-setuptool.sh -- install-ipfs --init --offline # offline ipfs daemon for testing
+        echo "ipfs = true" >> /home/rootless/.config/containerd-stargz-grpc/config.toml
+        systemctl --user restart stargz-snapshotter.service
+        export IPFS_PATH="/home/rootless/.local/share/ipfs"
 	exec "$@"
 fi

README.md

@@ -281,6 +281,8 @@ Run a command in a new container.
 
 Usage: `nerdctl run [OPTIONS] IMAGE [COMMAND] [ARG...]`
 
+:nerd_face: `ipfs://` prefix can be used for `IMAGE` to pull it from IFPS. See [`/docs/ipfs.md`](./docs/ipfs.md) for details.
+
 Basic flags:
 - :whale: :window: `-i, --interactive`: Keep STDIN open even if not attached"
 - :whale: :window: `-t, --tty`: Allocate a pseudo-TTY
@@ -667,6 +669,8 @@ Pull an image from a registry.
 
 Usage: `nerdctl pull [OPTIONS] NAME[:TAG|@DIGEST]`
 
+:nerd_face: `ipfs://` prefix can be used for `IMAGE` to pull it from IFPS. See [`/docs/ipfs.md`](./docs/ipfs.md) for details.
+
 Flags:
 - :whale: `--platform=(amd64|arm64|...)`: Pull content for a specific platform
   - :nerd_face: Unlike Docker, this flag can be specified multiple times (`--platform=amd64 --platform=arm64`)
@@ -680,6 +684,8 @@ Push an image to a registry.
 
 Usage: `nerdctl push [OPTIONS] NAME[:TAG]`
 
+:nerd_face: `ipfs://` prefix can be used for `IMAGE` to push it to IFPS. See [`/docs/ipfs.md`](./docs/ipfs.md) for details.
+
 Flags:
 - :nerd_face: `--platform=(amd64|arm64|...)`: Push content for a specific platform
 - :nerd_face: `--all-platforms`: Push content for all platforms
@@ -1110,3 +1116,4 @@ Others:
 - [`./docs/freebsd.md`](./docs/freebsd.md):  Running FreeBSD jails
 - [`./docs/multi-platform.md`](./docs/multi-platform.md):  Multi-platform mode
 - [`./docs/experimental.md`](./docs/experimental.md):  Experimental features
+- [`./docs/ipfs.md`](./docs/ipfs.md): Distributing images on IPFS

cmd/nerdctl/commit.go

@@ -21,9 +21,9 @@ import (
 	"errors"
 	"fmt"
 
-	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/nerdctl/pkg/idutil/containerwalker"
 	"github.com/containerd/nerdctl/pkg/imgutil/commit"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
 
 	"github.com/spf13/cobra"
 )
@@ -85,7 +85,7 @@ func commitAction(cmd *cobra.Command, args []string) error {
 func newCommitOpts(cmd *cobra.Command, args []string) (*commit.Opts, error) {
 	rawRef := args[1]
 
-	named, err := refdocker.ParseDockerRef(rawRef)
+	named, err := referenceutil.ParseDockerRef(rawRef)
 	if err != nil {
 		return nil, err
 	}

cmd/nerdctl/compose.go

@@ -24,10 +24,12 @@ import (
 	"github.com/containerd/containerd"
 	"github.com/containerd/containerd/errdefs"
 	"github.com/containerd/containerd/platforms"
-	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/nerdctl/pkg/composer"
 	"github.com/containerd/nerdctl/pkg/imgutil"
+	"github.com/containerd/nerdctl/pkg/ipfs"
 	"github.com/containerd/nerdctl/pkg/netutil"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
+	httpapi "github.com/ipfs/go-ipfs-http-client"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 
 	"github.com/spf13/cobra"
@@ -145,11 +147,11 @@ func getComposer(cmd *cobra.Command, client *containerd.Client) (*composer.Compo
 	}
 
 	o.ImageExists = func(ctx context.Context, rawRef string) (bool, error) {
-		named, err := refdocker.ParseDockerRef(rawRef)
+		refNamed, err := referenceutil.ParseAny(rawRef)
 		if err != nil {
 			return false, err
 		}
-		ref := named.String()
+		ref := refNamed.String()
 		if _, err := client.ImageService().Get(ctx, ref); err != nil {
 			if errors.Is(err, errdefs.ErrNotFound) {
 				return false, nil
@@ -168,8 +170,18 @@ func getComposer(cmd *cobra.Command, client *containerd.Client) (*composer.Compo
 			}
 			ocispecPlatforms = []ocispec.Platform{parsed} // no append
 		}
-		_, imgErr := imgutil.EnsureImage(ctx, client, cmd.OutOrStdout(), snapshotter, imageName,
-			pullMode, insecure, ocispecPlatforms, nil)
+		var imgErr error
+		if scheme, ref, err := referenceutil.ParseIPFSRefWithScheme(imageName); err == nil {
+			ipfsClient, err := httpapi.NewLocalApi()
+			if err != nil {
+				return err
+			}
+			_, imgErr = ipfs.EnsureImage(ctx, client, ipfsClient, cmd.OutOrStdout(), snapshotter, scheme, ref,
+				pullMode, ocispecPlatforms, nil)
+		} else {
+			_, imgErr = imgutil.EnsureImage(ctx, client, cmd.OutOrStdout(), snapshotter, imageName,
+				pullMode, insecure, ocispecPlatforms, nil)
+		}
 		return imgErr
 	}
 

cmd/nerdctl/compose_up_test.go

@@ -19,6 +19,7 @@ package main
 import (
 	"fmt"
 	"io"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -30,8 +31,73 @@ import (
 
 func TestComposeUp(t *testing.T) {
 	base := testutil.NewBase(t)
+	testComposeUp(t, base, fmt.Sprintf(`
+version: '3.1'
 
-	var dockerComposeYAML = fmt.Sprintf(`
+services:
+
+  wordpress:
+    image: %s
+    restart: always
+    ports:
+      - 8080:80
+    environment:
+      WORDPRESS_DB_HOST: db
+      WORDPRESS_DB_USER: exampleuser
+      WORDPRESS_DB_PASSWORD: examplepass
+      WORDPRESS_DB_NAME: exampledb
+    volumes:
+      - wordpress:/var/www/html
+
+  db:
+    image: %s
+    restart: always
+    environment:
+      MYSQL_DATABASE: exampledb
+      MYSQL_USER: exampleuser
+      MYSQL_PASSWORD: examplepass
+      MYSQL_RANDOM_ROOT_PASSWORD: '1'
+    volumes:
+      - db:/var/lib/mysql
+
+volumes:
+  wordpress:
+  db:
+`, testutil.WordpressImage, testutil.MariaDBImage))
+}
+
+func TestIPFSComposeUp(t *testing.T) {
+	requiresIPFS(t)
+	testutil.DockerIncompatible(t)
+	tests := []struct {
+		name           string
+		snapshotter    string
+		pushOptions    []string
+		requiresStargz bool
+	}{
+		{
+			name:        "overlayfs",
+			snapshotter: "overlayfs",
+		},
+		{
+			name:           "stargz",
+			snapshotter:    "stargz",
+			pushOptions:    []string{"--estargz"},
+			requiresStargz: true,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			base := testutil.NewBase(t)
+			if tt.requiresStargz {
+				requiresStargz(base)
+			}
+			ipfsImgs := make([]string, 2)
+			for i, img := range []string{testutil.WordpressImage, testutil.MariaDBImage} {
+				ipfsImgs[i] = pushImageToIPFS(t, base, img, tt.pushOptions...)
+			}
+			base.Env = append(os.Environ(), "CONTAINERD_SNAPSHOTTER="+tt.snapshotter)
+			testComposeUp(t, base, fmt.Sprintf(`
 version: '3.1'
 
 services:
@@ -47,6 +113,8 @@ services:
       WORDPRESS_DB_PASSWORD: examplepass
       WORDPRESS_DB_NAME: exampledb
     volumes:
+      # workaround for https://github.com/containerd/stargz-snapshotter/issues/444
+      - "/run"
       - wordpress:/var/www/html
 
   db:
@@ -58,12 +126,19 @@ services:
       MYSQL_PASSWORD: examplepass
       MYSQL_RANDOM_ROOT_PASSWORD: '1'
     volumes:
+      # workaround for https://github.com/containerd/stargz-snapshotter/issues/444
+      - "/run"
       - db:/var/lib/mysql
 
 volumes:
   wordpress:
   db:
-`, testutil.WordpressImage, testutil.MariaDBImage)
+`, ipfsImgs[0], ipfsImgs[1]))
+		})
+	}
+}
+
+func testComposeUp(t *testing.T, base *testutil.Base, dockerComposeYAML string) {
 	comp := testutil.NewComposeDir(t, dockerComposeYAML)
 	defer comp.CleanUp()
 

cmd/nerdctl/image_convert.go

@@ -25,8 +25,8 @@ import (
 
 	"github.com/containerd/containerd/images/converter"
 	"github.com/containerd/containerd/images/converter/uncompress"
-	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/nerdctl/pkg/platformutil"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
 	"github.com/containerd/stargz-snapshotter/estargz"
 	estargzconvert "github.com/containerd/stargz-snapshotter/nativeconverter/estargz"
 	"github.com/containerd/stargz-snapshotter/recorder"
@@ -90,13 +90,13 @@ func imageConvertAction(cmd *cobra.Command, args []string) error {
 		return errors.New("src and target image need to be specified")
 	}
 
-	srcNamed, err := refdocker.ParseDockerRef(srcRawRef)
+	srcNamed, err := referenceutil.ParseAny(srcRawRef)
 	if err != nil {
 		return err
 	}
 	srcRef := srcNamed.String()
 
-	targetNamed, err := refdocker.ParseDockerRef(targetRawRef)
+	targetNamed, err := referenceutil.ParseDockerRef(targetRawRef)
 	if err != nil {
 		return err
 	}

cmd/nerdctl/image_cryptutil.go

@@ -23,10 +23,10 @@ import (
 
 	"github.com/containerd/containerd/content"
 	"github.com/containerd/containerd/images/converter"
-	refdocker "github.com/containerd/containerd/reference/docker"
 	"github.com/containerd/imgcrypt/images/encryption"
 	"github.com/containerd/imgcrypt/images/encryption/parsehelpers"
 	"github.com/containerd/nerdctl/pkg/platformutil"
+	"github.com/containerd/nerdctl/pkg/referenceutil"
 	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	"github.com/spf13/cobra"
 )
@@ -105,13 +105,13 @@ func getImgcryptAction(encrypt bool) func(cmd *cobra.Command, args []string) err
 			return errors.New("src and target image need to be specified")
 		}
 
-		srcNamed, err := refdocker.ParseDockerRef(srcRawRef)
+		srcNamed, err := referenceutil.ParseAny(srcRawRef)
 		if err != nil {
 			return err
 		}
 		srcRef := srcNamed.String()
 
-		targetNamed, err := refdocker.ParseDockerRef(targetRawRef)
+		targetNamed, err := referenceutil.ParseDockerRef(targetRawRef)
 		if err != nil {
 			return err
 		}