test: add utilities and improvements for local testing

README.md

@@ -271,6 +271,8 @@ docker compose exec boost /bin/bash
 root@83260455bbd2:/app# ./sample/make-a-deal.sh
 ```
 
+You can also generate, dense, random cars and automatically make deals by leveraging the script at `./docker/devnet/boost/sample/random-deal.sh`. See the scripts comments for usage details.
+
 ### Accessing Lotus from localhost
 
 By default the [docker-compose.yaml](./docker-compose.yaml) does not expose any port of the `lotus` container. To access the `lotus` from a local machine:

cmd/boostx/main.go

@@ -33,6 +33,7 @@ func main() {
 		Commands: []*cli.Command{
 			commpCmd,
 			generatecarCmd,
+			generateRandCar,
 			marketAddCmd,
 			marketWithdrawCmd,
 			statsCmd,

cmd/boostx/utils_cmd.go

@@ -5,12 +5,15 @@ import (
 	"fmt"
 	"io"
 	"os"
+	"path"
 	"path/filepath"
+	"time"
 
 	clinode "github.com/filecoin-project/boost/cli/node"
 	"github.com/filecoin-project/boost/cmd"
 	"github.com/filecoin-project/boost/node"
 	"github.com/filecoin-project/boost/node/config"
+	"github.com/filecoin-project/boost/testutil"
 	"github.com/filecoin-project/go-commp-utils/writer"
 	"github.com/filecoin-project/go-fil-markets/stores"
 	"github.com/filecoin-project/go-state-types/abi"
@@ -34,6 +37,8 @@ import (
 	"github.com/ipfs/go-datastore/namespace"
 	ds_sync "github.com/ipfs/go-datastore/sync"
 	"github.com/ipld/go-car"
+	carv2 "github.com/ipld/go-car/v2"
+	"github.com/ipld/go-car/v2/blockstore"
 	selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
 	"github.com/multiformats/go-multibase"
 	"github.com/urfave/cli/v2"
@@ -237,6 +242,77 @@ var commpCmd = &cli.Command{
 	},
 }
 
+var generateRandCar = &cli.Command{
+	Name:      "generate-rand-car",
+	Usage:     "creates a randomly generated dense car",
+	ArgsUsage: "<outputPath>",
+	Before:    before,
+	Flags: []cli.Flag{
+		&cli.IntFlag{
+			Name:    "size",
+			Aliases: []string{"s"},
+			Usage:   "The size of the data to turn into a car",
+			Value:   8000000,
+		},
+		&cli.IntFlag{
+			Name:    "chunksize",
+			Aliases: []string{"c"},
+			Value:   512,
+			Usage:   "Size of chunking that should occur",
+		},
+		&cli.IntFlag{
+			Name:    "maxlinks",
+			Aliases: []string{"l"},
+			Value:   8,
+			Usage:   "Max number of leaves per level",
+		},
+	},
+	Action: func(cctx *cli.Context) error {
+		if cctx.Args().Len() < 1 {
+			return fmt.Errorf("usage: generate-car <outputPath> -size -chunksize -maxleaves")
+		}
+
+		outPath := cctx.Args().Get(0)
+		size := cctx.Int("size")
+		cs := cctx.Int64("chunksize")
+		ml := cctx.Int("maxlinks")
+
+		rf, err := testutil.CreateRandomFile(outPath, int(time.Now().Unix()), size)
+		if err != nil {
+			return err
+		}
+
+		// carv1
+		caropts := []carv2.Option{
+			blockstore.WriteAsCarV1(true),
+		}
+
+		root, cn, err := testutil.CreateDenseCARWith(outPath, rf, cs, ml, caropts)
+		if err != nil {
+			return err
+		}
+
+		err = os.Remove(rf)
+		if err != nil {
+			return err
+		}
+
+		encoder := cidenc.Encoder{Base: multibase.MustNewEncoder(multibase.Base32)}
+		rn := encoder.Encode(root)
+		base := path.Dir(cn)
+		np := path.Join(base, rn+".car")
+
+		err = os.Rename(cn, np)
+		if err != nil {
+			return err
+		}
+
+		fmt.Printf("Payload CID: %s, written to: %s\n", rn, np)
+
+		return nil
+	},
+}
+
 var generatecarCmd = &cli.Command{
 	Name:      "generate-car",
 	Usage:     "",

docker/devnet/boost/sample/random-deal.sh

@@ -0,0 +1,31 @@
+#!/usr/bin/env bash
+###################################################################################
+# Generates a random, dense car file in the /app/public/ directory and issues
+# a deal for it. This command assumes you have already transfered the needed funds.
+# Example (from ./docker/devnet):
+# $ docker compose exec boost /bin/bash
+# $ export `lotus auth api-info --perm=admin`
+# $ boost init
+# $ lotus send --from=`lotus wallet default` `boost wallet default` 100
+# $ boostx market-add 50
+# $ ./sample/random-deal.sh
+###################################################################################
+set -e
+
+ci="\e[3m"
+cn="\e[0m"
+
+FILE=`boostx generate-rand-car -c=512 -l=8 -s=5120000 /app/public/ | awk '{print $NF}'`
+PAYLOAD_CID=$(find "$FILE" | xargs -I{} basename {} | sed 's/\.car//')
+
+COMMP_CID=`boostx commp $FILE 2> /dev/null | grep CID | cut -d: -f2 | xargs`
+PIECE=`boostx commp $FILE 2> /dev/null | grep Piece | cut -d: -f2 | xargs`
+CAR=`boostx commp $FILE 2> /dev/null | grep Car | cut -d: -f2 | xargs`
+
+###################################################################################
+printf "${ci}boost deal --verified=false --provider=t01000 \
+--http-url=http://demo-http-server/$FILE \
+--commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \
+--payload-cid=$PAYLOAD_CID --storage-price 20000000000\n\n${cn}"
+
+boost deal --verified=false --provider=t01000 --http-url=http://demo-http-server/$PAYLOAD_CID.car --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --payload-cid=$PAYLOAD_CID --storage-price 20000000000

docker/devnet/docker-compose.yaml

@@ -54,6 +54,7 @@ services:
     ports:
       - "8080:8080"
       - "1288:1288" # For the /metrics endpoint
+      - "50000:50000" # Exposed libp2p port
     environment:
       - LOTUS_API_LISTENADDRESS=/dns/boost/tcp/1288/http
       - LOTUS_PATH=/var/lib/lotus
@@ -84,6 +85,9 @@ services:
       - ./data/boost:/var/lib/boost:ro
       - ./data/lotus:/var/lib/lotus:ro
       - ./data/lotus-miner:/var/lib/lotus-miner:ro
+    labels:
+      - "com.docker-tc.enabled=1"
+      - "com.docker-tc.delay=10ms"
 
   booster-bitswap:
     container_name: booster-bitswap
@@ -102,6 +106,9 @@ services:
       - ./data/boost:/var/lib/boost:ro
       - ./data/lotus:/var/lib/lotus:ro
       - ./data/lotus-miner:/var/lib/lotus-miner:ro
+    labels:
+      - "com.docker-tc.enabled=1"
+      - "com.docker-tc.delay=10ms"
 
   demo-http-server:
     container_name: demo-http-server
@@ -110,3 +117,24 @@ services:
     logging: *default-logging
     volumes:
       - ./data/sample:/usr/share/nginx/html:ro
+    labels:
+      - "com.docker-tc.enabled=1"
+      - "com.docker-tc.limit=1mbps"
+      - "com.docker-tc.delay=100ms"
+
+  tc:
+    image: "${DOCKER_IMAGE_TERMINAL:-lukaszlach/docker-tc}"
+    container_name: docker-tc
+    cap_add:
+      - NET_ADMIN
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock
+      - /var/docker-tc:/var/docker-tc
+    deploy:
+      mode: global
+      restart_policy:
+        condition: any
+    environment:
+      HTTP_BIND: "${HTTP_BIND:-127.0.0.1}"
+      HTTP_PORT: "${HTTP_PORT:-4080}"
+    network_mode: host

testutil/car.go

@@ -20,6 +20,7 @@ import (
 	"github.com/ipfs/go-merkledag"
 	"github.com/ipfs/go-unixfs/importer/balanced"
 	ihelper "github.com/ipfs/go-unixfs/importer/helpers"
+	"github.com/ipld/go-car/v2"
 	"github.com/ipld/go-car/v2/blockstore"
 	"github.com/multiformats/go-multihash"
 )
@@ -61,13 +62,23 @@ func CreateRandomFile(dir string, rseed, size int) (string, error) {
 	return file.Name(), nil
 }
 
+func CreateDenseCARv2(dir, src string) (cid.Cid, string, error) {
+	cs := int64(unixfsChunkSize)
+	maxlinks := unixfsLinksPerLevel
+	// Use carv2
+	caropts := []car.Option{
+		blockstore.UseWholeCIDs(true),
+	}
+	return CreateDenseCARWith(dir, src, cs, maxlinks, caropts)
+}
+
 // CreateDenseCARv2 generates a "dense" UnixFS CARv2 from the supplied ordinary file.
 // A dense UnixFS CARv2 is one storing leaf data. Contrast to CreateRefCARv2.
-func CreateDenseCARv2(dir, src string) (cid.Cid, string, error) {
+func CreateDenseCARWith(dir, src string, chunksize int64, maxlinks int, caropts []car.Option) (cid.Cid, string, error) {
 	bs := bstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore()))
 	dagSvc := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs)))
 
-	root, err := WriteUnixfsDAGTo(src, dagSvc)
+	root, err := WriteUnixfsDAGTo(src, dagSvc, chunksize, maxlinks)
 	if err != nil {
 		return cid.Undef, "", err
 	}
@@ -83,14 +94,14 @@ func CreateDenseCARv2(dir, src string) (cid.Cid, string, error) {
 		return cid.Undef, "", err
 	}
 
-	rw, err := blockstore.OpenReadWrite(out.Name(), []cid.Cid{root}, blockstore.UseWholeCIDs(true))
+	rw, err := blockstore.OpenReadWrite(out.Name(), []cid.Cid{root}, caropts...)
 	if err != nil {
 		return cid.Undef, "", err
 	}
 
 	dagSvc = merkledag.NewDAGService(blockservice.New(rw, offline.Exchange(rw)))
 
-	root2, err := WriteUnixfsDAGTo(src, dagSvc)
+	root2, err := WriteUnixfsDAGTo(src, dagSvc, chunksize, maxlinks)
 	if err != nil {
 		return cid.Undef, "", err
 	}
@@ -107,7 +118,7 @@ func CreateDenseCARv2(dir, src string) (cid.Cid, string, error) {
 	return root, out.Name(), nil
 }
 
-func WriteUnixfsDAGTo(path string, into ipldformat.DAGService) (cid.Cid, error) {
+func WriteUnixfsDAGTo(path string, into ipldformat.DAGService, chunksize int64, maxlinks int) (cid.Cid, error) {
 	file, err := os.Open(path)
 	if err != nil {
 		return cid.Undef, err
@@ -137,7 +148,7 @@ func WriteUnixfsDAGTo(path string, into ipldformat.DAGService) (cid.Cid, error)
 
 	bufferedDS := ipldformat.NewBufferedDAG(context.Background(), into)
 	params := ihelper.DagBuilderParams{
-		Maxlinks:  unixfsLinksPerLevel,
+		Maxlinks:  maxlinks,
 		RawLeaves: true,
 		// NOTE: InlineBuilder not recommended, we are using this to test identity CIDs
 		CidBuilder: cidutil.InlineBuilder{
@@ -148,7 +159,7 @@ func WriteUnixfsDAGTo(path string, into ipldformat.DAGService) (cid.Cid, error)
 		NoCopy:  true,
 	}
 
-	db, err := params.New(chunk.NewSizeSplitter(rpf, int64(unixfsChunkSize)))
+	db, err := params.New(chunk.NewSizeSplitter(rpf, chunksize))
 	if err != nil {
 		return cid.Undef, err
 	}