feat(shed): lotus-shed sectors dump-sectors (#12703)

Author: rvagg

cmd/lotus-shed/sectors.go

@@ -29,6 +29,8 @@ import (
 
 	"github.com/filecoin-project/lotus/api/v0api"
 	"github.com/filecoin-project/lotus/chain/actors"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/miner"
+	"github.com/filecoin-project/lotus/chain/actors/builtin/power"
 	"github.com/filecoin-project/lotus/chain/types"
 	lcli "github.com/filecoin-project/lotus/cli"
 	"github.com/filecoin-project/lotus/cli/spcli"
@@ -49,6 +51,7 @@ var sectorsCmd = &cli.Command{
 		terminateSectorPenaltyEstimationCmd,
 		visAllocatedSectorsCmd,
 		dumpRLESectorCmd,
+		dumpSectorOnChainInfoCmd,
 		sectorReadCmd,
 		sectorDeleteCmd,
 	},
@@ -652,6 +655,92 @@ var sectorDeleteCmd = &cli.Command{
 	},
 }
 
+var dumpSectorOnChainInfoCmd = &cli.Command{
+	Name:  "dump-sectors",
+	Usage: "Dump SectorOnChainInfo in CSV format for all sectors for all miners that have claimed power",
+	Action: func(cctx *cli.Context) error {
+		ctx := lcli.ReqContext(cctx)
+
+		h, err := loadChainStore(ctx, cctx.String("repo"))
+		if err != nil {
+			return xerrors.Errorf("loading chainstore: %w", err)
+		}
+		defer h.closer()
+
+		ts, err := lcli.LoadTipSet(ctx, cctx, &ChainStoreTipSetResolver{Chain: h.cs})
+		if err != nil {
+			return xerrors.Errorf("loading tipset: %w", err)
+		}
+
+		powerActor, err := h.sm.LoadActor(ctx, power.Address, ts)
+		if err != nil {
+			return xerrors.Errorf("failed to load power actor: %w", err)
+		}
+
+		powerState, err := power.Load(h.cs.ActorStore(ctx), powerActor)
+		if err != nil {
+			return xerrors.Errorf("failed to load power actor state: %w", err)
+		}
+
+		_, _ = fmt.Fprintf(cctx.App.Writer,
+			"Miner,SectorNumber,SealProof,DealIDCount,Activation,Expiration,DealWeight,VerifiedDealWeight,"+
+				"InitialPledge,ExpectedDayReward,ExpectedStoragePledge,PowerBaseEpoch,Flags\n")
+
+		var count int
+		err = powerState.ForEachClaim(func(maddr address.Address, claim power.Claim) error {
+			act, err := h.sm.LoadActorTsk(ctx, maddr, ts.Key())
+			if err != nil {
+				return xerrors.Errorf("failed to load miner actor: %w", err)
+			}
+
+			mas, err := miner.Load(h.sm.ChainStore().ActorStore(ctx), act)
+			if err != nil {
+				return xerrors.Errorf("failed to load miner actor state: %w", err)
+			}
+
+			soci, err := mas.LoadSectors(nil)
+			if err != nil {
+				return xerrors.Errorf("load sectors: %w", err)
+			}
+
+			for _, sector := range soci {
+				_, _ = fmt.Fprintf(
+					cctx.App.Writer,
+					"%s,%d,%d,%d,%d,%d,%s,%s,%s,%s,%s,%d,%x\n",
+					maddr,
+					sector.SectorNumber,
+					sector.SealProof,
+					len(sector.DealIDs),
+					sector.Activation,
+					sector.Expiration,
+					sector.DealWeight,
+					sector.VerifiedDealWeight,
+					sector.InitialPledge,
+					sector.ExpectedDayReward,
+					sector.ExpectedStoragePledge,
+					sector.PowerBaseEpoch,
+					sector.Flags,
+				)
+			}
+
+			count++
+			if count%1000 == 0 {
+				_, _ = fmt.Fprintf(cctx.App.ErrWriter, "Processed %d miners.\n", count)
+			}
+
+			return nil
+		})
+
+		if err != nil {
+			return xerrors.Errorf("iterating over claims: %w", err)
+		}
+
+		_, _ = fmt.Fprintf(cctx.App.ErrWriter, "Processed %d miners. Complete.\n", count)
+
+		return nil
+	},
+}
+
 type emptyLocalStorage struct {
 }
 

cmd/lotus-shed/state-stats.go

@@ -5,8 +5,9 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"io"
+	"os"
 	"path"
+	"path/filepath"
 	"reflect"
 	"sort"
 	"sync"
@@ -29,6 +30,8 @@ import (
 
 	"github.com/filecoin-project/lotus/api"
 	"github.com/filecoin-project/lotus/blockstore"
+	badgerbs "github.com/filecoin-project/lotus/blockstore/badger"
+	"github.com/filecoin-project/lotus/blockstore/splitstore"
 	"github.com/filecoin-project/lotus/chain/actors"
 	"github.com/filecoin-project/lotus/chain/actors/builtin"
 	"github.com/filecoin-project/lotus/chain/consensus"
@@ -226,41 +229,70 @@ func loadChainStore(ctx context.Context, repoPath string) (*StoreHandle, error)
 		return nil, xerrors.Errorf("lotus repo doesn't exist")
 	}
 
-	lr, err := r.Lock(repo.FullNode)
+	lkrepo, err := r.Lock(repo.FullNode)
 	if err != nil {
 		return nil, err
 	}
 
-	bs, err := lr.Blockstore(ctx, repo.UniversalBlockstore)
+	cold, err := lkrepo.Blockstore(ctx, repo.UniversalBlockstore)
 	if err != nil {
-		return nil, fmt.Errorf("failed to open blockstore: %w", err)
+		return nil, xerrors.Errorf("failed to open universal blockstore %w", err)
 	}
 
+	path, err := lkrepo.SplitstorePath()
+	if err != nil {
+		return nil, xerrors.Errorf("failed to get splitstore path: %w", err)
+	}
+
+	path = filepath.Join(path, "hot.badger")
+	if err := os.MkdirAll(path, 0755); err != nil {
+		return nil, xerrors.Errorf("failed to create hot.badger dir: %w", err)
+	}
+
+	opts, err := repo.BadgerBlockstoreOptions(repo.HotBlockstore, path, lkrepo.Readonly())
+	if err != nil {
+		return nil, xerrors.Errorf("failed to get badger blockstore options: %w", err)
+	}
+
+	hot, err := badgerbs.Open(opts)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to open badger blockstore: %w", err)
+	}
+
+	mds, err := lkrepo.Datastore(context.Background(), "/metadata")
+	if err != nil {
+		return nil, xerrors.Errorf("failed to open metadata datastore: %w", err)
+	}
+
+	cfg := &splitstore.Config{
+		MarkSetType:       "map",
+		DiscardColdBlocks: true,
+	}
+	ss, err := splitstore.Open(path, mds, hot, cold, cfg)
+	if err != nil {
+		return nil, xerrors.Errorf("failed to open splitstore: %w", err)
+	}
+
+	bs := ss
+
 	closer := func() {
-		if err := lr.Close(); err != nil {
+		if err := lkrepo.Close(); err != nil {
 			log.Warnf("failed to close locked repo: %s", err)
 		}
-		if c, ok := bs.(io.Closer); ok {
-			if err := c.Close(); err != nil {
-				log.Warnf("failed to close blockstore: %s", err)
-			}
+		if err := ss.Close(); err != nil {
+			log.Warnf("failed to close blockstore: %s", err)
 		}
 	}
 
-	mds, err := lr.Datastore(context.Background(), "/metadata")
-	if err != nil {
-		return nil, err
-	}
-
-	cs := store.NewChainStore(bs, bs, mds, nil, nil)
+	cs := store.NewChainStore(bs, bs, mds, filcns.Weight, nil)
 	if err := cs.Load(ctx); err != nil {
-		return nil, fmt.Errorf("failed to load chain store: %w", err)
+		return nil, xerrors.Errorf("failed to load chain store: %w", err)
 	}
 
 	tsExec := consensus.NewTipSetExecutor(filcns.RewardFunc)
 	sm, err := stmgr.NewStateManager(cs, tsExec, vm.Syscalls(proofsffi.ProofVerifier), filcns.DefaultUpgradeSchedule(), nil, mds, nil)
 	if err != nil {
-		return nil, fmt.Errorf("failed to open state manager: %w", err)
+		return nil, xerrors.Errorf("failed to open state manager: %w", err)
 	}
 	handle := StoreHandle{
 		bs:     bs,