Merge pull request #9013 from filecoin-project/feat/path-type-filters

feat: storage: Path type filters

.circleci/config.yml

@@ -903,6 +903,11 @@ workflows:
           suite: itest-nonce
           target: "./itests/nonce_test.go"
 
+      - test:
+          name: test-itest-path_type_filters
+          suite: itest-path_type_filters
+          target: "./itests/path_type_filters_test.go"
+
       - test:
           name: test-itest-paych_api
           suite: itest-paych_api
@@ -978,6 +983,11 @@ workflows:
           suite: itest-wdpost_dispute
           target: "./itests/wdpost_dispute_test.go"
 
+      - test:
+          name: test-itest-wdpost_no_miner_storage
+          suite: itest-wdpost_no_miner_storage
+          target: "./itests/wdpost_no_miner_storage_test.go"
+
       - test:
           name: test-itest-wdpost
           suite: itest-wdpost

api/api_storage.go

@@ -146,18 +146,29 @@ type StorageMiner interface {
 	SealingSchedDiag(ctx context.Context, doSched bool) (interface{}, error) //perm:admin
 	SealingAbort(ctx context.Context, call storiface.CallID) error           //perm:admin
 
-	// SectorIndex
-	StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error                                                                                             //perm:admin
-	StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error)                                                                                              //perm:admin
-	StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error                                                                                       //perm:admin
-	StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error                                     //perm:admin
-	StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error                                                      //perm:admin
+	// paths.SectorIndex
+	StorageAttach(context.Context, storiface.StorageInfo, fsutil.FsStat) error                                                         //perm:admin
+	StorageInfo(context.Context, storiface.ID) (storiface.StorageInfo, error)                                                          //perm:admin
+	StorageReportHealth(context.Context, storiface.ID, storiface.HealthReport) error                                                   //perm:admin
+	StorageDeclareSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType, primary bool) error //perm:admin
+	StorageDropSector(ctx context.Context, storageID storiface.ID, s abi.SectorID, ft storiface.SectorFileType) error                  //perm:admin
+	// StorageFindSector returns list of paths where the specified sector files exist.
+	//
+	// If allowFetch is set, list of paths to which the sector can be fetched will also be returned.
+	// - Paths which have sector files locally (don't require fetching) will be listed first.
+	// - Paths which have sector files locally will not be filtered based on based on AllowTypes/DenyTypes.
+	// - Paths which require fetching will be filtered based on AllowTypes/DenyTypes. If multiple
+	//   file types are specified, each type will be considered individually, and a union of all paths
+	//   which can accommodate each file type will be returned.
 	StorageFindSector(ctx context.Context, sector abi.SectorID, ft storiface.SectorFileType, ssize abi.SectorSize, allowFetch bool) ([]storiface.SectorStorageInfo, error) //perm:admin
-	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error)           //perm:admin
-	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                             //perm:admin
-	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                                  //perm:admin
-	StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error)                                                                                            //perm:admin
-	StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)                                                                                                    //perm:admin
+	// StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.
+	// Paths with more weight and more % of free space are preferred.
+	// Note: This method doesn't filter paths based on AllowTypes/DenyTypes.
+	StorageBestAlloc(ctx context.Context, allocate storiface.SectorFileType, ssize abi.SectorSize, pathType storiface.PathType) ([]storiface.StorageInfo, error) //perm:admin
+	StorageLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) error                                   //perm:admin
+	StorageTryLock(ctx context.Context, sector abi.SectorID, read storiface.SectorFileType, write storiface.SectorFileType) (bool, error)                        //perm:admin
+	StorageList(ctx context.Context) (map[storiface.ID][]storiface.Decl, error)                                                                                  //perm:admin
+	StorageGetLocks(ctx context.Context) (storiface.SectorLocks, error)                                                                                          //perm:admin
 
 	StorageLocal(ctx context.Context) (map[storiface.ID]string, error)       //perm:admin
 	StorageStat(ctx context.Context, id storiface.ID) (fsutil.FsStat, error) //perm:admin

cmd/lotus-miner/init.go

@@ -458,7 +458,7 @@ func storageMinerInit(ctx context.Context, cctx *cli.Context, api v1api.FullNode
 			wsts := statestore.New(namespace.Wrap(mds, modules.WorkerCallsPrefix))
 			smsts := statestore.New(namespace.Wrap(mds, modules.ManagerWorkPrefix))
 
-			si := paths.NewIndex()
+			si := paths.NewIndex(nil)
 
 			lstor, err := paths.NewLocal(ctx, lr, si, nil)
 			if err != nil {

cmd/lotus-miner/storage.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"math/bits"
 	"os"
 	"path/filepath"
 	"sort"
@@ -345,6 +346,20 @@ var storageListCmd = &cli.Command{
 				fmt.Printf("\tAllowTo: %s\n", strings.Join(si.AllowTo, ", "))
 			}
 
+			if len(si.AllowTypes) > 0 || len(si.DenyTypes) > 0 {
+				denied := storiface.FTAll.SubAllowed(si.AllowTypes, si.DenyTypes)
+				allowed := storiface.FTAll ^ denied
+
+				switch {
+				case bits.OnesCount64(uint64(allowed)) == 0:
+					fmt.Printf("\tAllow Types: %s\n", color.RedString("None"))
+				case bits.OnesCount64(uint64(allowed)) < bits.OnesCount64(uint64(denied)):
+					fmt.Printf("\tAllow Types: %s\n", color.GreenString(strings.Join(allowed.Strings(), " ")))
+				default:
+					fmt.Printf("\tDeny Types:  %s\n", color.RedString(strings.Join(denied.Strings(), " ")))
+				}
+			}
+
 			if localPath, ok := local[s.ID]; ok {
 				fmt.Printf("\tLocal: %s\n", color.GreenString(localPath))
 			}

documentation/en/api-v0-methods-miner.md

@@ -3271,7 +3271,7 @@ Inputs:
 Response: `{}`
 
 ### StorageAttach
-SectorIndex
+paths.SectorIndex
 
 
 Perms: admin
@@ -3293,6 +3293,12 @@ Inputs:
     ],
     "AllowTo": [
       "string value"
+    ],
+    "AllowTypes": [
+      "string value"
+    ],
+    "DenyTypes": [
+      "string value"
     ]
   },
   {
@@ -3328,6 +3334,9 @@ Response:
 ```
 
 ### StorageBestAlloc
+StorageBestAlloc returns list of paths where sector files of the specified type can be allocated, ordered by preference.
+Paths with more weight and more % of free space are preferred.
+Note: This method doesn't filter paths based on AllowTypes/DenyTypes.
 
 
 Perms: admin
@@ -3358,6 +3367,12 @@ Response:
     ],
     "AllowTo": [
       "string value"
+    ],
+    "AllowTypes": [
+      "string value"
+    ],
+    "DenyTypes": [
+      "string value"
     ]
   }
 ]
@@ -3403,6 +3418,14 @@ Inputs:
 Response: `{}`
 
 ### StorageFindSector
+StorageFindSector returns list of paths where the specified sector files exist.
+
+If allowFetch is set, list of paths to which the sector can be fetched will also be returned.
+- Paths which have sector files locally (don't require fetching) will be listed first.
+- Paths which have sector files locally will not be filtered based on based on AllowTypes/DenyTypes.
+- Paths which require fetching will be filtered based on AllowTypes/DenyTypes. If multiple
+  file types are specified, each type will be considered individually, and a union of all paths
+  which can accommodate each file type will be returned.
 
 
 Perms: admin
@@ -3434,7 +3457,13 @@ Response:
     "Weight": 42,
     "CanSeal": true,
     "CanStore": true,
-    "Primary": true
+    "Primary": true,
+    "AllowTypes": [
+      "string value"
+    ],
+    "DenyTypes": [
+      "string value"
+    ]
   }
 ]
 ```
@@ -3502,6 +3531,12 @@ Response:
   ],
   "AllowTo": [
     "string value"
+  ],
+  "AllowTypes": [
+    "string value"
+  ],
+  "DenyTypes": [
+    "string value"
   ]
 }
 ```

itests/kit/blockminer.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"strings"
 	"sync"
 	"sync/atomic"
 	"testing"
@@ -184,7 +185,12 @@ func (bm *BlockMiner) MineBlocksMustPost(ctx context.Context, blocktime time.Dur
 
 			var target abi.ChainEpoch
 			reportSuccessFn := func(success bool, epoch abi.ChainEpoch, err error) {
-				require.NoError(bm.t, err)
+				// if api shuts down before mining, we may get an error which we should probably just ignore
+				// (fixing it will require rewriting most of the mining loop)
+				if err != nil && !strings.Contains(err.Error(), "websocket connection closed") {
+					require.NoError(bm.t, err)
+				}
+
 				target = epoch
 				wait <- success
 			}

itests/kit/ensemble.go

@@ -546,7 +546,12 @@ func (n *Ensemble) Start() *Ensemble {
 		// using real proofs, therefore need real sectors.
 		if !n.bootstrapped && !n.options.mockProofs {
 			psd := m.PresealDir
+			noPaths := m.options.noStorage
+
 			err := lr.SetStorage(func(sc *paths.StorageConfig) {
+				if noPaths {
+					sc.StoragePaths = []paths.LocalPath{}
+				}
 				sc.StoragePaths = append(sc.StoragePaths, paths.LocalPath{Path: psd})
 			})
 
@@ -693,6 +698,13 @@ func (n *Ensemble) Start() *Ensemble {
 		lr, err := r.Lock(repo.Worker)
 		require.NoError(n.t, err)
 
+		if m.options.noStorage {
+			err := lr.SetStorage(func(sc *paths.StorageConfig) {
+				sc.StoragePaths = []paths.LocalPath{}
+			})
+			require.NoError(n.t, err)
+		}
+
 		ds, err := lr.Datastore(context.Background(), "/metadata")
 		require.NoError(n.t, err)
 

itests/kit/node_miner.go

@@ -167,28 +167,29 @@ func (tm *TestMiner) FlushSealingBatches(ctx context.Context) {
 
 const metaFile = "sectorstore.json"
 
-func (tm *TestMiner) AddStorage(ctx context.Context, t *testing.T, weight uint64, seal, store bool) {
-	p, err := ioutil.TempDir("", "lotus-testsectors-")
-	require.NoError(t, err)
+func (tm *TestMiner) AddStorage(ctx context.Context, t *testing.T, conf func(*paths.LocalStorageMeta)) storiface.ID {
+	p := t.TempDir()
 
 	if err := os.MkdirAll(p, 0755); err != nil {
 		if !os.IsExist(err) {
 			require.NoError(t, err)
 		}
 	}
 
-	_, err = os.Stat(filepath.Join(p, metaFile))
+	_, err := os.Stat(filepath.Join(p, metaFile))
 	if !os.IsNotExist(err) {
 		require.NoError(t, err)
 	}
 
 	cfg := &paths.LocalStorageMeta{
 		ID:       storiface.ID(uuid.New().String()),
-		Weight:   weight,
-		CanSeal:  seal,
-		CanStore: store,
+		Weight:   10,
+		CanSeal:  false,
+		CanStore: false,
 	}
 
+	conf(cfg)
+
 	if !(cfg.CanStore || cfg.CanSeal) {
 		t.Fatal("must specify at least one of CanStore or cfg.CanSeal")
 	}
@@ -201,4 +202,6 @@ func (tm *TestMiner) AddStorage(ctx context.Context, t *testing.T, weight uint64
 
 	err = tm.StorageAddLocal(ctx, p)
 	require.NoError(t, err)
+
+	return cfg.ID
 }

itests/kit/node_opts.go

@@ -43,6 +43,7 @@ type nodeOpts struct {
 	minerNoLocalSealing    bool // use worker
 	minerAssigner          string
 	disallowRemoteFinalize bool
+	noStorage              bool
 
 	workerTasks      []sealtasks.TaskType
 	workerStorageOpt func(paths.Store) paths.Store
@@ -154,6 +155,14 @@ func PresealSectors(sectors int) NodeOpt {
 	}
 }
 
+// NoStorage initializes miners with no writable storage paths (just read-only preseal paths)
+func NoStorage() NodeOpt {
+	return func(opts *nodeOpts) error {
+		opts.noStorage = true
+		return nil
+	}
+}
+
 // ThroughRPC makes interactions with this node throughout the test flow through
 // the JSON-RPC API.
 func ThroughRPC() NodeOpt {
@@ -210,6 +219,8 @@ func WithTaskTypes(tt []sealtasks.TaskType) NodeOpt {
 	}
 }
 
+var WithSealWorkerTasks = WithTaskTypes([]sealtasks.TaskType{sealtasks.TTFetch, sealtasks.TTCommit1, sealtasks.TTFinalize, sealtasks.TTAddPiece, sealtasks.TTPreCommit1, sealtasks.TTPreCommit2, sealtasks.TTCommit2, sealtasks.TTUnseal})
+
 func WithWorkerStorage(transform func(paths.Store) paths.Store) NodeOpt {
 	return func(opts *nodeOpts) error {
 		opts.workerStorageOpt = transform