Fix #1691: Error when adding directories with many files

Fixes #1691 by updating to the latest go-unixfs and adding a test.

The test is verified to fail on the previous go-unixfs version.

adder/adder_test.go

@@ -3,6 +3,7 @@ package adder
 import (
 	"bytes"
 	"context"
+	"fmt"
 	"mime/multipart"
 	"sync"
 	"testing"
@@ -23,7 +24,14 @@ type mockCDAGServ struct {
 
 func newMockCDAGServ() *mockCDAGServ {
 	return &mockCDAGServ{
-		MockDAGService: test.NewMockDAGService(),
+		// write-only DAGs.
+		MockDAGService: test.NewMockDAGService(true),
+	}
+}
+
+func newReadableMockCDAGServ() *mockCDAGServ {
+	return &mockCDAGServ{
+		MockDAGService: test.NewMockDAGService(false),
 	}
 }
 
@@ -145,13 +153,14 @@ func TestAdder_CAR(t *testing.T) {
 	defer closer.Close()
 	r := multipart.NewReader(mr, mr.Boundary())
 	p := api.DefaultAddParams()
-	dags := newMockCDAGServ()
+	dags := newReadableMockCDAGServ()
 	adder := New(dags, p, nil)
 	root, err := adder.FromMultipart(ctx, r)
 	if err != nil {
 		t.Fatal(err)
 	}
 	var carBuf bytes.Buffer
+	// Make a CAR out of the files we added.
 	err = car.WriteCar(ctx, dags, []cid.Cid{root.Cid}, &carBuf)
 	if err != nil {
 		t.Fatal(err)
@@ -188,3 +197,31 @@ func TestAdder_CAR(t *testing.T) {
 	}
 
 }
+
+func TestAdder_LargeFolder(t *testing.T) {
+	items := 10000 // add 10000 items
+
+	sth := test.NewShardingTestHelper()
+	defer sth.Clean(t)
+
+	filesMap := make(map[string]files.Node)
+	for i := 0; i < items; i++ {
+		fstr := fmt.Sprintf("file%d", i)
+		f := files.NewBytesFile([]byte(fstr))
+		filesMap[fstr] = f
+	}
+
+	slf := files.NewMapDirectory(filesMap)
+
+	p := api.DefaultAddParams()
+	p.Wrap = true
+
+	dags := newMockCDAGServ()
+
+	adder := New(dags, p, nil)
+	_, err := adder.FromFiles(context.Background(), slf)
+
+	if err != nil {
+		t.Fatal(err)
+	}
+}

adder/util.go

@@ -3,7 +3,6 @@ package adder
 import (
 	"context"
 	"errors"
-	"fmt"
 	"sync"
 
 	"github.com/ipfs-cluster/ipfs-cluster/api"
@@ -165,7 +164,7 @@ func (dag BaseDAGService) Get(ctx context.Context, key cid.Cid) (ipld.Node, erro
 // GetMany returns an output channel that always emits an error
 func (dag BaseDAGService) GetMany(ctx context.Context, keys []cid.Cid) <-chan *ipld.NodeOption {
 	out := make(chan *ipld.NodeOption, 1)
-	out <- &ipld.NodeOption{Err: fmt.Errorf("failed to fetch all nodes")}
+	out <- &ipld.NodeOption{Err: errors.New("failed to fetch all nodes")}
 	close(out)
 	return out
 }

go.mod

@@ -36,7 +36,7 @@ require (
 	github.com/ipfs/go-merkledag v0.6.0
 	github.com/ipfs/go-mfs v0.1.3-0.20210507195338-96fbfa122164
 	github.com/ipfs/go-path v0.3.0
-	github.com/ipfs/go-unixfs v0.3.1
+	github.com/ipfs/go-unixfs v0.4.0
 	github.com/ipld/go-car v0.3.3
 	github.com/kelseyhightower/envconfig v1.4.0
 	github.com/kishansagathiya/go-dot v0.1.0

go.sum

@@ -602,6 +602,10 @@ github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMU
 github.com/ipfs/go-unixfs v0.2.6/go.mod h1:GTTzQvaZsTZARdNkkdjDKFFnBhmO3e5mIM1PkH/x4p0=
 github.com/ipfs/go-unixfs v0.3.1 h1:LrfED0OGfG98ZEegO4/xiprx2O+yS+krCMQSp7zLVv8=
 github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/pCFf1o=
+github.com/ipfs/go-unixfs v0.3.2-0.20220615181143-9ac1b9ff0917 h1:TzNOzSioqAfd6VQS8+PWAnKH4M7shjyDVe6iSvQO7HE=
+github.com/ipfs/go-unixfs v0.3.2-0.20220615181143-9ac1b9ff0917/go.mod h1:I7Nqtm06HgOOd+setAoCU6rf/HgVFHE+peeNuOv/5+g=
+github.com/ipfs/go-unixfs v0.4.0 h1:qSyyxfB/OiDdWHYiSbyaqKC7zfSE/TFL0QdwkRjBm20=
+github.com/ipfs/go-unixfs v0.4.0/go.mod h1:I7Nqtm06HgOOd+setAoCU6rf/HgVFHE+peeNuOv/5+g=
 github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s=
 github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E=
 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0=