diff --git a/README.md b/README.md index e1ab7ceb5..7bd93925b 100644 --- a/README.md +++ b/README.md @@ -271,6 +271,8 @@ docker compose exec boost /bin/bash root@83260455bbd2:/app# ./sample/make-a-deal.sh ``` +You can also generate, dense, random cars and automatically make deals by leveraging the script at `./docker/devnet/boost/sample/random-deal.sh`. See the scripts comments for usage details. + ### Accessing Lotus from localhost By default the [docker-compose.yaml](./docker-compose.yaml) does not expose any port of the `lotus` container. To access the `lotus` from a local machine: diff --git a/cmd/boostx/main.go b/cmd/boostx/main.go index c5247f04a..443976750 100644 --- a/cmd/boostx/main.go +++ b/cmd/boostx/main.go @@ -33,6 +33,7 @@ func main() { Commands: []*cli.Command{ commpCmd, generatecarCmd, + generateRandCar, marketAddCmd, marketWithdrawCmd, statsCmd, diff --git a/cmd/boostx/utils_cmd.go b/cmd/boostx/utils_cmd.go index 7fb72db56..042b23786 100644 --- a/cmd/boostx/utils_cmd.go +++ b/cmd/boostx/utils_cmd.go @@ -5,12 +5,15 @@ import ( "fmt" "io" "os" + "path" "path/filepath" + "time" clinode "github.com/filecoin-project/boost/cli/node" "github.com/filecoin-project/boost/cmd" "github.com/filecoin-project/boost/node" "github.com/filecoin-project/boost/node/config" + "github.com/filecoin-project/boost/testutil" "github.com/filecoin-project/go-commp-utils/writer" "github.com/filecoin-project/go-fil-markets/stores" "github.com/filecoin-project/go-state-types/abi" @@ -34,6 +37,8 @@ import ( "github.com/ipfs/go-datastore/namespace" ds_sync "github.com/ipfs/go-datastore/sync" "github.com/ipld/go-car" + carv2 "github.com/ipld/go-car/v2" + "github.com/ipld/go-car/v2/blockstore" selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse" "github.com/multiformats/go-multibase" "github.com/urfave/cli/v2" @@ -237,6 +242,77 @@ var commpCmd = &cli.Command{ }, } +var generateRandCar = &cli.Command{ + Name: "generate-rand-car", + Usage: "creates a randomly generated dense car", + ArgsUsage: "", + Before: before, + Flags: []cli.Flag{ + &cli.IntFlag{ + Name: "size", + Aliases: []string{"s"}, + Usage: "The size of the data to turn into a car", + Value: 8000000, + }, + &cli.IntFlag{ + Name: "chunksize", + Aliases: []string{"c"}, + Value: 512, + Usage: "Size of chunking that should occur", + }, + &cli.IntFlag{ + Name: "maxlinks", + Aliases: []string{"l"}, + Value: 8, + Usage: "Max number of leaves per level", + }, + }, + Action: func(cctx *cli.Context) error { + if cctx.Args().Len() < 1 { + return fmt.Errorf("usage: generate-car -size -chunksize -maxleaves") + } + + outPath := cctx.Args().Get(0) + size := cctx.Int("size") + cs := cctx.Int64("chunksize") + ml := cctx.Int("maxlinks") + + rf, err := testutil.CreateRandomFile(outPath, int(time.Now().Unix()), size) + if err != nil { + return err + } + + // carv1 + caropts := []carv2.Option{ + blockstore.WriteAsCarV1(true), + } + + root, cn, err := testutil.CreateDenseCARWith(outPath, rf, cs, ml, caropts) + if err != nil { + return err + } + + err = os.Remove(rf) + if err != nil { + return err + } + + encoder := cidenc.Encoder{Base: multibase.MustNewEncoder(multibase.Base32)} + rn := encoder.Encode(root) + base := path.Dir(cn) + np := path.Join(base, rn+".car") + + err = os.Rename(cn, np) + if err != nil { + return err + } + + fmt.Printf("Payload CID: %s, written to: %s\n", rn, np) + + return nil + }, +} + var generatecarCmd = &cli.Command{ Name: "generate-car", Usage: "", diff --git a/docker/devnet/boost/sample/random-deal.sh b/docker/devnet/boost/sample/random-deal.sh new file mode 100755 index 000000000..502fff7f9 --- /dev/null +++ b/docker/devnet/boost/sample/random-deal.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +################################################################################### +# Generates a random, dense car file in the /app/public/ directory and issues +# a deal for it. This command assumes you have already transfered the needed funds. +# Example (from ./docker/devnet): +# $ docker compose exec boost /bin/bash +# $ export `lotus auth api-info --perm=admin` +# $ boost init +# $ lotus send --from=`lotus wallet default` `boost wallet default` 100 +# $ boostx market-add 50 +# $ ./sample/random-deal.sh +################################################################################### +set -e + +ci="\e[3m" +cn="\e[0m" + +FILE=`boostx generate-rand-car -c=512 -l=8 -s=5120000 /app/public/ | awk '{print $NF}'` +PAYLOAD_CID=$(find "$FILE" | xargs -I{} basename {} | sed 's/\.car//') + +COMMP_CID=`boostx commp $FILE 2> /dev/null | grep CID | cut -d: -f2 | xargs` +PIECE=`boostx commp $FILE 2> /dev/null | grep Piece | cut -d: -f2 | xargs` +CAR=`boostx commp $FILE 2> /dev/null | grep Car | cut -d: -f2 | xargs` + +################################################################################### +printf "${ci}boost deal --verified=false --provider=t01000 \ +--http-url=http://demo-http-server/$FILE \ +--commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE \ +--payload-cid=$PAYLOAD_CID --storage-price 20000000000\n\n${cn}" + +boost deal --verified=false --provider=t01000 --http-url=http://demo-http-server/$PAYLOAD_CID.car --commp=$COMMP_CID --car-size=$CAR --piece-size=$PIECE --payload-cid=$PAYLOAD_CID --storage-price 20000000000 \ No newline at end of file diff --git a/docker/devnet/docker-compose.yaml b/docker/devnet/docker-compose.yaml index c65fd9cf3..07aab9235 100644 --- a/docker/devnet/docker-compose.yaml +++ b/docker/devnet/docker-compose.yaml @@ -54,6 +54,7 @@ services: ports: - "8080:8080" - "1288:1288" # For the /metrics endpoint + - "50000:50000" # Exposed libp2p port environment: - LOTUS_API_LISTENADDRESS=/dns/boost/tcp/1288/http - LOTUS_PATH=/var/lib/lotus @@ -84,6 +85,9 @@ services: - ./data/boost:/var/lib/boost:ro - ./data/lotus:/var/lib/lotus:ro - ./data/lotus-miner:/var/lib/lotus-miner:ro + labels: + - "com.docker-tc.enabled=1" + - "com.docker-tc.delay=10ms" booster-bitswap: container_name: booster-bitswap @@ -102,6 +106,9 @@ services: - ./data/boost:/var/lib/boost:ro - ./data/lotus:/var/lib/lotus:ro - ./data/lotus-miner:/var/lib/lotus-miner:ro + labels: + - "com.docker-tc.enabled=1" + - "com.docker-tc.delay=10ms" demo-http-server: container_name: demo-http-server @@ -110,3 +117,24 @@ services: logging: *default-logging volumes: - ./data/sample:/usr/share/nginx/html:ro + labels: + - "com.docker-tc.enabled=1" + - "com.docker-tc.limit=1mbps" + - "com.docker-tc.delay=100ms" + + tc: + image: "${DOCKER_IMAGE_TERMINAL:-lukaszlach/docker-tc}" + container_name: docker-tc + cap_add: + - NET_ADMIN + volumes: + - /var/run/docker.sock:/var/run/docker.sock + - /var/docker-tc:/var/docker-tc + deploy: + mode: global + restart_policy: + condition: any + environment: + HTTP_BIND: "${HTTP_BIND:-127.0.0.1}" + HTTP_PORT: "${HTTP_PORT:-4080}" + network_mode: host \ No newline at end of file diff --git a/testutil/car.go b/testutil/car.go index 3e85a72b7..9db0d69ff 100644 --- a/testutil/car.go +++ b/testutil/car.go @@ -20,6 +20,7 @@ import ( "github.com/ipfs/go-merkledag" "github.com/ipfs/go-unixfs/importer/balanced" ihelper "github.com/ipfs/go-unixfs/importer/helpers" + "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/blockstore" "github.com/multiformats/go-multihash" ) @@ -61,13 +62,23 @@ func CreateRandomFile(dir string, rseed, size int) (string, error) { return file.Name(), nil } +func CreateDenseCARv2(dir, src string) (cid.Cid, string, error) { + cs := int64(unixfsChunkSize) + maxlinks := unixfsLinksPerLevel + // Use carv2 + caropts := []car.Option{ + blockstore.UseWholeCIDs(true), + } + return CreateDenseCARWith(dir, src, cs, maxlinks, caropts) +} + // CreateDenseCARv2 generates a "dense" UnixFS CARv2 from the supplied ordinary file. // A dense UnixFS CARv2 is one storing leaf data. Contrast to CreateRefCARv2. -func CreateDenseCARv2(dir, src string) (cid.Cid, string, error) { +func CreateDenseCARWith(dir, src string, chunksize int64, maxlinks int, caropts []car.Option) (cid.Cid, string, error) { bs := bstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) dagSvc := merkledag.NewDAGService(blockservice.New(bs, offline.Exchange(bs))) - root, err := WriteUnixfsDAGTo(src, dagSvc) + root, err := WriteUnixfsDAGTo(src, dagSvc, chunksize, maxlinks) if err != nil { return cid.Undef, "", err } @@ -83,14 +94,14 @@ func CreateDenseCARv2(dir, src string) (cid.Cid, string, error) { return cid.Undef, "", err } - rw, err := blockstore.OpenReadWrite(out.Name(), []cid.Cid{root}, blockstore.UseWholeCIDs(true)) + rw, err := blockstore.OpenReadWrite(out.Name(), []cid.Cid{root}, caropts...) if err != nil { return cid.Undef, "", err } dagSvc = merkledag.NewDAGService(blockservice.New(rw, offline.Exchange(rw))) - root2, err := WriteUnixfsDAGTo(src, dagSvc) + root2, err := WriteUnixfsDAGTo(src, dagSvc, chunksize, maxlinks) if err != nil { return cid.Undef, "", err } @@ -107,7 +118,7 @@ func CreateDenseCARv2(dir, src string) (cid.Cid, string, error) { return root, out.Name(), nil } -func WriteUnixfsDAGTo(path string, into ipldformat.DAGService) (cid.Cid, error) { +func WriteUnixfsDAGTo(path string, into ipldformat.DAGService, chunksize int64, maxlinks int) (cid.Cid, error) { file, err := os.Open(path) if err != nil { return cid.Undef, err @@ -137,7 +148,7 @@ func WriteUnixfsDAGTo(path string, into ipldformat.DAGService) (cid.Cid, error) bufferedDS := ipldformat.NewBufferedDAG(context.Background(), into) params := ihelper.DagBuilderParams{ - Maxlinks: unixfsLinksPerLevel, + Maxlinks: maxlinks, RawLeaves: true, // NOTE: InlineBuilder not recommended, we are using this to test identity CIDs CidBuilder: cidutil.InlineBuilder{ @@ -148,7 +159,7 @@ func WriteUnixfsDAGTo(path string, into ipldformat.DAGService) (cid.Cid, error) NoCopy: true, } - db, err := params.New(chunk.NewSizeSplitter(rpf, int64(unixfsChunkSize))) + db, err := params.New(chunk.NewSizeSplitter(rpf, chunksize)) if err != nil { return cid.Undef, err }