feat: add multi-miner to single LID support (#1656)

* feat: retrieve data served from multiple miners (#1578)

* add booster-http multi-miner retrieve test (#1598)

* feat: add booster-http multi-miner retrieve test

* refactor: move api version check to its own function

* test: add booster-bitswap multi-miner test (#1599)

* multi-miner / single LID - piece doctor (#1600)

* wip: multi-miner / 1 LID - piece doctor

* refactor: move yugabyte migration details into a separate class

* fix: add miner address to yugabyte / leveldb impl

* refactor: update log in piece doctor no deals for miner to warning

* refactor: break migration into two parts to avoid migrations bug

* feat: skip set address in migration if PieceTracker table has no rows (#1614)

* fix: split PieceTracker and PieceFlagged migrations (#1615)

* fix: yugabyte migrate command name

* Add miner address to top of page in boost UI (#1633)

* feat: add miner address to top of page in boost UI

* Update react/src/MinerAddress.js

Co-authored-by: LexLuthr <88259624+LexLuthr@users.noreply.github.com>

---------

Co-authored-by: LexLuthr <88259624+LexLuthr@users.noreply.github.com>

* refactor: move StorageAccessApiInfo config into DealMaking at GraphsyncStorageAccessApiInfo (#1634)

* Show list of retrieval miners on retrievals list page (#1637)

* feat: show list of retrieval miners on retrievals list page

* fix: retrieval miner address resolver

* fix: mmlid fixes for LID UI (#1639)

* fix: look up unseal status against multi-miner accessor (rather than just on local miner) (#1641)

* Show warning on piece doctor page if still doing initial scan (#1643)

* fix: look up unseal status against multi-miner accessor (rather than just on local miner)

* feat: show warning on piece doctor page if still doing initial scan

* Update extern/boostd-data/yugabyte/piecedoctor.go

Co-authored-by: Anton Evangelatov <anton.evangelatov@gmail.com>

* Update extern/boostd-data/yugabyte/piecedoctor.go

---------

Co-authored-by: Anton Evangelatov <anton.evangelatov@gmail.com>

* feat: change postgres timezone type to include timezone (#1645)

* feat: show network name in binary versions (#1642)

* fix boostd-data maddr, enhance build

* add CQL migrator

* refactor migrator

* modify migrate query

* remove idx from migrate query

* remove CQL migrator

* undo go mod changes, fix mainnet build

* undo yugabyte changes

* remove extra line

* Cassandra migrations (#1648)

* feat: cassandra migrations

* refactor: dbname -> appliedMigration

* fix: yugabyte lid test

* feat: add postgres migration

* refactor: simplify cassandra migrations

* fix: delete duplicate rows in PieceTracker table

* fix test

---------

Co-authored-by: dirkmc <dirkmdev@gmail.com>
Co-authored-by: Anton Evangelatov <anton.evangelatov@gmail.com>

.circleci/config.yml

@@ -344,12 +344,27 @@ workflows:
       - test:
           name: test-all
           suite: all
-          target: "`go list ./... | grep -v boost/itests`"
+          target: "`go list ./... | grep -v boost/itests | grep -v cmd/booster-http | grep -v cmd/booster-bitswap`"
 
       - test:
           name: test-itest-ipni
           suite: itest-ipni
           target: "./itests/ipni_publish_test.go"
 
+      - test:
+          name: test-itest-multiminer-graphsync
+          suite: itest-multiminer-graphsync
+          target: "./itests/multiminer_retrieval_graphsync_test.go"
+
+      - test:
+          name: test-booster-http
+          suite: booster-http
+          target: "./cmd/booster-http"
+
+      - test:
+          name: test-booster-bitswap
+          suite: booster-bitswap
+          target: "./cmd/booster-bitswap"
+
       - lid-docker-compose
 

build/params_calibnet.go

@@ -0,0 +1,13 @@
+//go:build calibnet
+// +build calibnet
+
+package build
+
+import (
+	"github.com/filecoin-project/go-address"
+)
+
+func init() {
+	SetAddressNetwork(address.Testnet)
+	BuildType = BuildCalibnet
+}

build/params_mainnet.go

@@ -0,0 +1,13 @@
+//go:build !calibnet && !debug && !2k
+// +build !calibnet,!debug,!2k
+
+package build
+
+import (
+	"github.com/filecoin-project/go-address"
+)
+
+func init() {
+	SetAddressNetwork(address.Mainnet)
+	BuildType = BuildMainnet
+}

build/params_testnets.go

@@ -0,0 +1,13 @@
+//go:build debug || 2k
+// +build debug 2k
+
+package build
+
+import (
+	"github.com/filecoin-project/go-address"
+)
+
+func init() {
+	SetAddressNetwork(address.Testnet)
+	BuildType = BuildDebug
+}

build/version.go

@@ -1,9 +1,38 @@
 package build
 
 var CurrentCommit string
+var BuildType int
+
+const (
+	BuildMainnet      = 0x1
+	Build2k           = 0x2
+	BuildDebug        = 0x3
+	BuildCalibnet     = 0x4
+	BuildInteropnet   = 0x5
+	BuildButterflynet = 0x7
+)
+
+func BuildTypeString() string {
+	switch BuildType {
+	case BuildMainnet:
+		return "+mainnet"
+	case Build2k:
+		return "+2k"
+	case BuildDebug:
+		return "+debug"
+	case BuildCalibnet:
+		return "+calibnet"
+	case BuildInteropnet:
+		return "+interopnet"
+	case BuildButterflynet:
+		return "+butterflynet"
+	default:
+		return "+huh?"
+	}
+}
 
 const BuildVersion = "2.0.0-rc1"
 
 func UserVersion() string {
-	return BuildVersion + CurrentCommit
+	return BuildVersion + BuildTypeString() + CurrentCommit
 }

cmd/boostd/main.go

@@ -73,6 +73,7 @@ func before(cctx *cli.Context) error {
 	_ = logging.SetLogLevel("piecedoc", "INFO")
 	_ = logging.SetLogLevel("piecedirectory", "INFO")
 	_ = logging.SetLogLevel("sectorstatemgr", "INFO")
+	_ = logging.SetLogLevel("migrations", "INFO")
 
 	if cliutil.IsVeryVerbose {
 		_ = logging.SetLogLevel("boostd", "DEBUG")

cmd/boostd/recover.go

@@ -170,6 +170,11 @@ func action(cctx *cli.Context) error {
 	}
 	defer ncloser()
 
+	err = lib.CheckFullNodeApiVersion(ctx, fullnodeApi)
+	if err != nil {
+		return err
+	}
+
 	// Connect to the storage API and create a sector accessor
 	storageApiInfo := cctx.String("api-storage")
 

cmd/booster-bitswap/client.go

@@ -13,6 +13,7 @@ import (
 	"github.com/ipld/go-ipld-prime"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
 	"github.com/ipld/go-ipld-prime/traversal"
+	"github.com/libp2p/go-libp2p/core/host"
 	mh "github.com/multiformats/go-multihash"
 
 	"github.com/filecoin-project/boost/cmd/booster-bitswap/bitswap"
@@ -90,14 +91,7 @@ var fetchCmd = &cli.Command{
 			return err
 		}
 
-		host, err := libp2p.New(
-			libp2p.Transport(tcp.NewTCPTransport),
-			libp2p.Transport(quic.NewTransport),
-			libp2p.Muxer("/mplex/6.7.0", mplex.DefaultTransport),
-			libp2p.Muxer("/yamux/1.0.0", yamux.DefaultTransport),
-			libp2p.Identity(privKey),
-			libp2p.ResourceManager(&network.NullResourceManager{}),
-		)
+		host, err := createClientHost(privKey)
 		if err != nil {
 			return err
 		}
@@ -169,6 +163,17 @@ var fetchCmd = &cli.Command{
 	},
 }
 
+func createClientHost(privKey crypto.PrivKey) (host.Host, error) {
+	return libp2p.New(
+		libp2p.Transport(tcp.NewTCPTransport),
+		libp2p.Transport(quic.NewTransport),
+		libp2p.Muxer("/mplex/6.7.0", mplex.DefaultTransport),
+		libp2p.Muxer("/yamux/1.0.0", yamux.DefaultTransport),
+		libp2p.Identity(privKey),
+		libp2p.ResourceManager(&network.NullResourceManager{}),
+	)
+}
+
 func getBlocks(ctx context.Context, bsClient *client.Client, c cid.Cid, throttle chan struct{}) (uint64, uint64, error) {
 	var size uint64
 	var links []cid.Cid

cmd/booster-bitswap/main.go

@@ -18,22 +18,23 @@ var FlagRepo = &cli.StringFlag{
 	EnvVars: []string{"BOOSTER_BITSWAP_REPO"},
 }
 
+var app = &cli.App{
+	Name:                 "booster-bitswap",
+	Usage:                "Bitswap endpoint for retrieval from Filecoin",
+	EnableBashCompletion: true,
+	Version:              build.UserVersion(),
+	Flags: []cli.Flag{
+		cliutil.FlagVeryVerbose,
+		FlagRepo,
+	},
+	Commands: []*cli.Command{
+		initCmd,
+		runCmd,
+		fetchCmd,
+	},
+}
+
 func main() {
-	app := &cli.App{
-		Name:                 "booster-bitswap",
-		Usage:                "Bitswap endpoint for retrieval from Filecoin",
-		EnableBashCompletion: true,
-		Version:              build.UserVersion(),
-		Flags: []cli.Flag{
-			cliutil.FlagVeryVerbose,
-			FlagRepo,
-		},
-		Commands: []*cli.Command{
-			initCmd,
-			runCmd,
-			fetchCmd,
-		},
-	}
 	app.Setup()
 
 	if err := app.Run(os.Args); err != nil {

cmd/booster-bitswap/multiminer_retrieval_test.go

@@ -0,0 +1,134 @@
+package main
+
+import (
+	"context"
+	"crypto/rand"
+	"fmt"
+	"github.com/filecoin-project/boost/cmd/booster-bitswap/bitswap"
+	"github.com/filecoin-project/boost/itests/shared"
+	carv2 "github.com/ipld/go-car/v2"
+	"github.com/libp2p/go-libp2p/core/crypto"
+	"github.com/libp2p/go-libp2p/core/host"
+	"github.com/libp2p/go-libp2p/core/peer"
+	"github.com/multiformats/go-multiaddr"
+	"github.com/stretchr/testify/require"
+	"path"
+	"sort"
+	"testing"
+	"time"
+)
+
+func TestMultiMinerBitswapRetrieval(t *testing.T) {
+	shared.RunMultiminerRetrievalTest(t, func(ctx context.Context, t *testing.T, rt *shared.RetrievalTest) {
+		miner1ApiInfo, err := rt.BoostAndMiner1.LotusMinerApiInfo()
+		require.NoError(t, err)
+
+		miner2ApiInfo, err := rt.BoostAndMiner2.LotusMinerApiInfo()
+		require.NoError(t, err)
+
+		fullNode2ApiInfo, err := rt.BoostAndMiner2.LotusFullNodeApiInfo()
+		require.NoError(t, err)
+
+		repoDir := t.TempDir()
+		peerID, _, err := configureRepo(repoDir, true)
+		require.NoError(t, err)
+
+		runCtx, cancelRun := context.WithCancel(ctx)
+		defer cancelRun()
+
+		go func() {
+			// Configure booster-bitswap to
+			// - Get piece location information from the shared LID instance
+			// - Get the piece data from either miner1 or miner2 (depending on the location info)
+			apiInfo := []string{miner1ApiInfo, miner2ApiInfo}
+			_ = runBoosterBitswap(runCtx, repoDir, apiInfo, fullNode2ApiInfo, "ws://localhost:8042")
+		}()
+
+		maddr, err := multiaddr.NewMultiaddr("/ip4/127.0.0.1/tcp/8888")
+		require.NoError(t, err)
+
+		privKey, _, err := crypto.GenerateECDSAKeyPair(rand.Reader)
+		require.NoError(t, err)
+
+		clientHost, err := createClientHost(privKey)
+		require.NoError(t, err)
+
+		t.Logf("waiting for server to come up")
+		start := time.Now()
+		require.Eventually(t, func() bool {
+			err := connectToHost(ctx, clientHost, maddr, peerID)
+			if err != nil {
+				t.Logf("connecting to host: %s", err)
+				return false
+			}
+			return true
+		}, 30*time.Second, time.Second)
+		t.Logf("booster-bitswap is up after %s", time.Since(start))
+
+		outPath := path.Join(t.TempDir(), "out.dat")
+		fetchAddr := maddr.String() + "/p2p/" + peerID.String()
+		err = runBoosterBitswapFetch(ctx, fetchAddr, rt.RootCid.String(), outPath)
+		require.NoError(t, err)
+
+		t.Logf("retrieval is done, compare root cid %s to downloaded CAR root cid", rt.RootCid)
+		r, err := carv2.OpenReader(outPath)
+		require.NoError(t, err)
+
+		roots, err := r.Roots()
+		require.NoError(t, err)
+		require.Len(t, roots, 1)
+		require.Equal(t, rt.RootCid, roots[0])
+
+		t.Logf("file retrieved successfully")
+	})
+}
+
+func connectToHost(ctx context.Context, clientHost host.Host, maddr multiaddr.Multiaddr, pid peer.ID) error {
+	// Connect to host
+	err := clientHost.Connect(ctx, peer.AddrInfo{
+		ID:    pid,
+		Addrs: []multiaddr.Multiaddr{maddr},
+	})
+	if err != nil {
+		return err
+	}
+
+	// Check host's libp2p protocols
+	protos, err := clientHost.Peerstore().GetProtocols(pid)
+	if err != nil {
+		return fmt.Errorf("getting protocols from peer store for %s: %w", pid, err)
+	}
+	sort.Slice(protos, func(i, j int) bool {
+		return protos[i] < protos[j]
+	})
+	fmt.Println("host libp2p protocols", "protocols", protos)
+	p, err := clientHost.Peerstore().FirstSupportedProtocol(pid, bitswap.Protocols...)
+	if err != nil {
+		return fmt.Errorf("getting first supported protocol from peer store for %s: %w", pid, err)
+	}
+	if p == "" {
+		return fmt.Errorf("host %s does not support any know bitswap protocols: %s", pid, bitswap.ProtocolStrings)
+	}
+	return nil
+}
+
+func runBoosterBitswap(ctx context.Context, repo string, minerApiInfo []string, fullNodeApiInfo string, lidApiInfo string) error {
+	app.Setup()
+
+	args := []string{"booster-bitswap",
+		"--repo=" + repo,
+		"run",
+		"--api-fullnode=" + fullNodeApiInfo,
+		"--api-lid=" + lidApiInfo,
+		"--api-version-check=false",
+	}
+	for _, apiInfo := range minerApiInfo {
+		args = append(args, "--api-storage="+apiInfo)
+	}
+	return app.RunContext(ctx, args)
+}
+
+func runBoosterBitswapFetch(ctx context.Context, multiaddr string, rootCid string, outputPath string) error {
+	args := []string{"booster-bitswap", "fetch", multiaddr, rootCid, outputPath}
+	return app.RunContext(ctx, args)
+}