Merge pull request #735 from jmank88/url-nocopy

support --nocopy when adding URLs

adder/adder.go

@@ -117,6 +117,7 @@ func (a *Adder) FromFiles(ctx context.Context, f files.Directory) (cid.Cid, erro
 	ipfsAdder.Chunker = a.params.Chunker
 	ipfsAdder.Out = a.output
 	ipfsAdder.Progress = a.params.Progress
+	ipfsAdder.NoCopy = a.params.NoCopy
 
 	// Set up prefix
 	prefix, err := merkledag.PrefixForCidVersion(a.params.CidVersion)

api/add.go

@@ -37,6 +37,7 @@ type AddParams struct {
 	CidVersion     int
 	HashFun        string
 	StreamChannels bool
+	NoCopy         bool
 }
 
 // DefaultAddParams returns a AddParams object with standard defaults
@@ -53,6 +54,7 @@ func DefaultAddParams() *AddParams {
 		CidVersion:     0,
 		HashFun:        "sha2-256",
 		StreamChannels: true,
+		NoCopy:         false,
 		PinOptions: PinOptions{
 			ReplicationFactorMin: 0,
 			ReplicationFactorMax: 0,
@@ -162,6 +164,11 @@ func AddParamsFromQuery(query url.Values) (*AddParams, error) {
 		return nil, err
 	}
 
+	err = parseBoolParam(query, "nocopy", &params.NoCopy)
+	if err != nil {
+		return nil, err
+	}
+
 	return params, nil
 }
 
@@ -183,6 +190,7 @@ func (p *AddParams) ToQueryString() string {
 	query.Set("cid-version", fmt.Sprintf("%d", p.CidVersion))
 	query.Set("hash", p.HashFun)
 	query.Set("stream-channels", fmt.Sprintf("%t", p.StreamChannels))
+	query.Set("nocopy", fmt.Sprintf("%t", p.NoCopy))
 	return query.Encode()
 }
 
@@ -201,5 +209,6 @@ func (p *AddParams) Equals(p2 *AddParams) bool {
 		p.Wrap == p2.Wrap &&
 		p.CidVersion == p2.CidVersion &&
 		p.HashFun == p2.HashFun &&
-		p.StreamChannels == p2.StreamChannels
+		p.StreamChannels == p2.StreamChannels &&
+		p.NoCopy == p2.NoCopy
 }

api/rest/client/methods.go

@@ -516,6 +516,10 @@ func (c *defaultClient) Add(
 			addFile = files.NewWebFile(u)
 			name = path.Base(u.Path)
 		} else {
+			if params.NoCopy {
+				close(out)
+				return fmt.Errorf("nocopy option is only valid for URLs")
+			}
 			addFile, err = makeSerialFile(p, params)
 			if err != nil {
 				close(out)

cmd/ipfs-cluster-ctl/main.go

@@ -349,6 +349,10 @@ cluster "pin add".
 					Value: defaultAddParams.ReplicationFactorMax,
 					Usage: "Sets the maximum replication factor for pinning this file",
 				},
+				cli.BoolFlag{
+					Name:  "nocopy",
+					Usage: "Add the URL using filestore. Implies raw-leaves. (experimental)",
+				},
 				// TODO: Uncomment when sharding is supported.
 				// cli.BoolFlag{
 				//	Name:  "shard",
@@ -410,6 +414,10 @@ cluster "pin add".
 				if p.CidVersion > 0 {
 					p.RawLeaves = true
 				}
+				p.NoCopy = c.Bool("nocopy")
+				if p.NoCopy {
+					p.RawLeaves = true
+				}
 
 				out := make(chan *api.AddedOutput, 1)
 				var wg sync.WaitGroup

go.mod

@@ -21,7 +21,7 @@ require (
 	github.com/ipfs/go-ipfs-api v0.0.1
 	github.com/ipfs/go-ipfs-chunker v0.0.1
 	github.com/ipfs/go-ipfs-ds-help v0.0.1
-	github.com/ipfs/go-ipfs-files v0.0.1
+	github.com/ipfs/go-ipfs-files v0.0.2
 	github.com/ipfs/go-ipfs-posinfo v0.0.1
 	github.com/ipfs/go-ipfs-util v0.0.1
 	github.com/ipfs/go-ipld-cbor v0.0.1

go.sum

@@ -148,6 +148,8 @@ github.com/ipfs/go-ipfs-exchange-offline v0.0.1 h1:P56jYKZF7lDDOLx5SotVh5KFxoY6C
 github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0=
 github.com/ipfs/go-ipfs-files v0.0.1 h1:OroTsI58plHGX70HPLKy6LQhPR3HZJ5ip61fYlo6POM=
 github.com/ipfs/go-ipfs-files v0.0.1/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
+github.com/ipfs/go-ipfs-files v0.0.2 h1:fEEjF4H+1t8SFOHqUGp0KqcwgIRlbD2bu8CAS2sIggE=
+github.com/ipfs/go-ipfs-files v0.0.2/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4=
 github.com/ipfs/go-ipfs-flags v0.0.1 h1:OH5cEkJYL0QgA+bvD55TNG9ud8HA2Nqaav47b2c/UJk=
 github.com/ipfs/go-ipfs-flags v0.0.1/go.mod h1:RnXBb9WV53GSfTrSDVK61NLTFKvWc60n+K9EgCDh+rA=
 github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs=