Skip to content

Commit

Permalink
Merge pull request #511 from ktock/ipfsdeps
Browse files Browse the repository at this point in the history 
Avoid adding ipfs dependencies to `github.com/containerd/stargz-snapshotter`
  • Loading branch information
@AkihiroSuda
AkihiroSuda committed Nov 4, 2021
2 parents 20c9e08 + abde477 commit 6b6ca38
Show file tree
Hide file tree
Showing 22 changed files with 3,959 additions and 1,027 deletions.
22 changes: 16 additions & 6 deletions Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
# Base path used to install.
CMD_DESTDIR ?= /usr/local
GO111MODULE_VALUE=auto
PREFIX ?= out/
PREFIX ?= $(CURDIR)/out/

PKG=github.com/containerd/stargz-snapshotter
VERSION=$(shell git describe --match 'v[0-9]*' --dirty='.m' --always --tags)
Expand All @@ -27,7 +27,7 @@ CMD=containerd-stargz-grpc ctr-remote stargz-store

CMD_BINARIES=$(addprefix $(PREFIX),$(CMD))

.PHONY: all build check install-check-tools install uninstall clean test test-root test-all integration test-optimize benchmark test-kind test-cri-containerd test-cri-o test-criauth generate validate-generated test-k3s test-k3s-argo-workflow
.PHONY: all build check install-check-tools install uninstall clean test test-root test-all integration test-optimize benchmark test-kind test-cri-containerd test-cri-o test-criauth generate validate-generated test-k3s test-k3s-argo-workflow vendor

all: build

Expand All @@ -36,18 +36,20 @@ build: $(CMD)
FORCE:

containerd-stargz-grpc: FORCE
GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/containerd-stargz-grpc
cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./containerd-stargz-grpc

ctr-remote: FORCE
GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/ctr-remote
cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./ctr-remote

stargz-store: FORCE
GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./cmd/stargz-store
cd cmd/ ; GO111MODULE=$(GO111MODULE_VALUE) go build -o $(PREFIX)$@ $(GO_BUILD_FLAGS) $(GO_LD_FLAGS) -v ./stargz-store

check:
@echo "$@"
@GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
@cd ./cmd ; GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run
@cd ./ipfs ; GO111MODULE=$(GO111MODULE_VALUE) $(shell go env GOPATH)/bin/golangci-lint run

install-check-tools:
@curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(shell go env GOPATH)/bin v1.42.1
Expand All @@ -71,10 +73,18 @@ generate:
validate-generated:
@./script/generated-files/generate.sh validate

vendor:
@GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
@cd ./cmd ; GO111MODULE=$(GO111MODULE_VALUE) go mod tidy
@cd ./ipfs ; GO111MODULE=$(GO111MODULE_VALUE) go mod tidy

test:
@echo "$@"
@GO111MODULE=$(GO111MODULE_VALUE) go test -race ./...
@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) go test -timeout 20m -race ./...
@cd ./estargz ; GO111MODULE=$(GO111MODULE_VALUE) go test -timeout 30m -race ./...
@cd ./cmd ; GO111MODULE=$(GO111MODULE_VALUE) go test -timeout 20m -race ./...
@cd ./ipfs ; GO111MODULE=$(GO111MODULE_VALUE) go test -timeout 20m -race ./...

test-root:
@echo "$@"
Expand Down
38 changes: 18 additions & 20 deletions fs/remote/ipfs/reader.go → ...ainerd-stargz-grpc/ipfs/resolvehandler.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,24 +22,17 @@ import (
"fmt"
"io"

"github.com/containerd/stargz-snapshotter/fs/remote"
"github.com/containerd/stargz-snapshotter/ipfs"
httpapi "github.com/ipfs/go-ipfs-http-client"
iface "github.com/ipfs/interface-go-ipfs-core"
ipath "github.com/ipfs/interface-go-ipfs-core/path"
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
)

type Reader struct {
api iface.CoreAPI
path ipath.Path
}

func Supported(desc ocispec.Descriptor) bool {
_, err := ipfs.GetPath(desc)
return err == nil
}
type ResolveHandler struct{}

func NewReader(ctx context.Context, desc ocispec.Descriptor) (*Reader, int64, error) {
func (r *ResolveHandler) Handle(ctx context.Context, desc ocispec.Descriptor) (remote.Fetcher, int64, error) {
p, err := ipfs.GetPath(desc)
if err != nil {
return nil, 0, err
Expand All @@ -62,11 +55,16 @@ func NewReader(ctx context.Context, desc ocispec.Descriptor) (*Reader, int64, er
if err != nil {
return nil, 0, err
}
return &Reader{client, p}, s, nil
return &fetcher{client, p}, s, nil
}

type fetcher struct {
api iface.CoreAPI
path ipath.Path
}

func (r *Reader) Reader(ctx context.Context, off int64, size int64) (io.ReadCloser, error) {
n, err := r.api.Unixfs().Get(ctx, r.path)
func (f *fetcher) Fetch(ctx context.Context, off int64, size int64) (io.ReadCloser, error) {
n, err := f.api.Unixfs().Get(ctx, f.path)
if err != nil {
return nil, err
}
Expand All @@ -76,14 +74,14 @@ func (r *Reader) Reader(ctx context.Context, off int64, size int64) (io.ReadClos
if !ok {
return nil, fmt.Errorf("ReaderAt is not implemented")
}
return &reader{
return &readCloser{
Reader: io.NewSectionReader(ra, off, size),
closeFunc: n.Close,
}, nil
}

func (r *Reader) Check() error {
n, err := r.api.Unixfs().Get(context.Background(), r.path)
func (f *fetcher) Check() error {
n, err := f.api.Unixfs().Get(context.Background(), f.path)
if err != nil {
return err
}
Expand All @@ -95,14 +93,14 @@ func (r *Reader) Check() error {
return n.Close()
}

func (r *Reader) GenID(off int64, size int64) string {
sum := sha256.Sum256([]byte(fmt.Sprintf("%s-%d-%d", r.path.String(), off, size)))
func (f *fetcher) GenID(off int64, size int64) string {
sum := sha256.Sum256([]byte(fmt.Sprintf("%s-%d-%d", f.path.String(), off, size)))
return fmt.Sprintf("%x", sum)
}

type reader struct {
type readCloser struct {
io.Reader
closeFunc func() error
}

func (r *reader) Close() error { return r.closeFunc() }
func (r *readCloser) Close() error { return r.closeFunc() }
12 changes: 11 additions & 1 deletion cmd/containerd-stargz-grpc/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,8 @@ import (
"github.com/containerd/containerd/pkg/dialer"
"github.com/containerd/containerd/snapshots"
"github.com/containerd/containerd/sys"
ipfs "github.com/containerd/stargz-snapshotter/cmd/containerd-stargz-grpc/ipfs"
"github.com/containerd/stargz-snapshotter/fs"
"github.com/containerd/stargz-snapshotter/service"
"github.com/containerd/stargz-snapshotter/service/keychain/cri"
"github.com/containerd/stargz-snapshotter/service/keychain/dockerconfig"
Expand Down Expand Up @@ -80,6 +82,9 @@ type snapshotterConfig struct {

// DebugAddress is a Unix domain socket address where the snapshotter exposes /debug/ endpoints.
DebugAddress string `toml:"debug_address"`

// IPFS is a flag to enbale lazy pulling from IPFS.
IPFS bool `toml:"ipfs"`
}

func main() {
Expand Down Expand Up @@ -162,7 +167,12 @@ func main() {
runtime.RegisterImageServiceServer(rpc, criServer)
credsFuncs = append(credsFuncs, f)
}
rs, err := service.NewStargzSnapshotterService(ctx, *rootDir, &config.Config, service.WithCredsFuncs(credsFuncs...))
var fsOpts []fs.Option
if config.IPFS {
fsOpts = append(fsOpts, fs.WithResolveHandler("ipfs", new(ipfs.ResolveHandler)))
}
rs, err := service.NewStargzSnapshotterService(ctx, *rootDir, &config.Config,
service.WithCredsFuncs(credsFuncs...), service.WithFilesystemOptions(fsOpts...))
if err != nil {
log.G(ctx).WithError(err).Fatalf("failed to configure snapshotter")
}
Expand Down
37 changes: 37 additions & 0 deletions cmd/go.mod
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,37 @@
module github.com/containerd/stargz-snapshotter/cmd

go 1.16

require (
github.com/containerd/containerd v1.5.7
github.com/containerd/go-cni v1.1.0
github.com/containerd/stargz-snapshotter v0.9.0
github.com/containerd/stargz-snapshotter/estargz v0.9.0
github.com/containerd/stargz-snapshotter/ipfs v0.9.0
github.com/coreos/go-systemd/v22 v22.3.2
github.com/docker/go-metrics v0.0.1
github.com/hashicorp/go-multierror v1.1.1
github.com/ipfs/go-ipfs-http-client v0.1.0
github.com/ipfs/interface-go-ipfs-core v0.5.2
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.2-0.20210819154149-5ad6f50d6283
github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417
github.com/pelletier/go-toml v1.9.4
github.com/pkg/errors v0.9.1
github.com/rs/xid v1.3.0
github.com/sirupsen/logrus v1.8.1
github.com/urfave/cli v1.22.4
golang.org/x/sys v0.0.0-20210915083310-ed5796bab164
google.golang.org/grpc v1.42.0
k8s.io/cri-api v0.22.3
)

replace (
// Import local packages.
github.com/containerd/stargz-snapshotter => ../
github.com/containerd/stargz-snapshotter/estargz => ../estargz
github.com/containerd/stargz-snapshotter/ipfs => ../ipfs

// Temporary fork for avoiding importing patent-protected code: https://github.com/hashicorp/golang-lru/issues/73
github.com/hashicorp/golang-lru => github.com/ktock/golang-lru v0.5.5-0.20211029085301-ec551be6f75c
)
Loading

0 comments on commit 6b6ca38

Please sign in to comment.