fix: allow client to set fast-retrieval

api/api_full.go

@@ -2,7 +2,6 @@ package api
 
 import (
 	"context"
-	"encoding/json"
 	"fmt"
 	"time"
 
@@ -297,34 +296,6 @@ type MethodCall struct {
 	Error string
 }
 
-type StartDealParams struct {
-	Data               *storagemarket.DataRef
-	Wallet             address.Address
-	Miner              address.Address
-	EpochPrice         types.BigInt
-	MinBlocksDuration  uint64
-	ProviderCollateral big.Int
-	DealStartEpoch     abi.ChainEpoch
-	FastRetrieval      bool
-	VerifiedDeal       bool
-}
-
-func (s *StartDealParams) UnmarshalJSON(raw []byte) (err error) {
-	type sdpAlias StartDealParams
-
-	sdp := sdpAlias{
-		FastRetrieval: true,
-	}
-
-	if err := json.Unmarshal(raw, &sdp); err != nil {
-		return err
-	}
-
-	*s = StartDealParams(sdp)
-
-	return nil
-}
-
 type IpldObject struct {
 	Cid cid.Cid
 	Obj interface{}

cmd/boost/deal_cmd.go

@@ -54,8 +54,8 @@ var dealFlags = []cli.Flag{
 		Required: true,
 	},
 	&cli.IntFlag{
-		Name:  "start-epoch",
-		Usage: "start epoch by when the deal should be proved by provider on-chain",
+		Name:        "start-epoch",
+		Usage:       "start epoch by when the deal should be proved by provider on-chain",
 		DefaultText: "current chain head + 2 days",
 	},
 	&cli.IntFlag{
@@ -77,6 +77,11 @@ var dealFlags = []cli.Flag{
 		Usage: "whether the deal funds should come from verified client data-cap",
 		Value: true,
 	},
+	&cli.BoolFlag{
+		Name:  "fast-retrieval",
+		Usage: "indicates that data should be available for fast retrieval",
+		Value: true,
+	},
 	&cli.StringFlag{
 		Name:  "wallet",
 		Usage: "wallet address to be used to initiate the deal",
@@ -251,6 +256,7 @@ func dealCmdAction(cctx *cli.Context, isOnline bool) error {
 		DealDataRoot:       rootCid,
 		IsOffline:          !isOnline,
 		Transfer:           transfer,
+		FastRetrieval:      cctx.Bool("fast-retrieval"),
 	}
 
 	log.Debugw("about to submit deal proposal", "uuid", dealUuid.String())

db/deals.go

@@ -78,6 +78,7 @@ func newDealAccessor(db *sql.DB, deal *types.ProviderDealState) *dealAccessor {
 			"CheckpointAt":          &fielddef.FieldDef{F: &deal.CheckpointAt},
 			"Error":                 &fielddef.FieldDef{F: &deal.Err},
 			"Retry":                 &fielddef.FieldDef{F: &deal.Retry},
+			"FastRetrieval":         &fielddef.FieldDef{F: &deal.FastRetrieval},
 
 			// Needed so the deal can be looked up by signed proposal cid
 			"SignedProposalCID": &fielddef.SignedPropFieldDef{Prop: deal.ClientDealProposal},

db/fixtures.go

@@ -84,14 +84,15 @@ func GenerateNDeals(count int) ([]types.ProviderDealState, error) {
 					Params: []byte(fmt.Sprintf(`{"url":"http://files.org/file%d.car"}`, rand.Intn(1000))),
 					Size:   uint64(rand.Intn(10000)),
 				},
-				ChainDealID: abi.DealID(rand.Intn(10000)),
-				PublishCID:  &publishCid,
-				SectorID:    abi.SectorNumber(rand.Intn(10000)),
-				Offset:      abi.PaddedPieceSize(rand.Intn(1000000)),
-				Length:      abi.PaddedPieceSize(rand.Intn(1000000)),
-				Checkpoint:  dealcheckpoints.Accepted,
-				Retry:       types.DealRetryAuto,
-				Err:         dealErr,
+				ChainDealID:   abi.DealID(rand.Intn(10000)),
+				PublishCID:    &publishCid,
+				SectorID:      abi.SectorNumber(rand.Intn(10000)),
+				Offset:        abi.PaddedPieceSize(rand.Intn(1000000)),
+				Length:        abi.PaddedPieceSize(rand.Intn(1000000)),
+				Checkpoint:    dealcheckpoints.Accepted,
+				Retry:         types.DealRetryAuto,
+				Err:           dealErr,
+				FastRetrieval: false,
 			}
 
 			deals = append(deals, deal)

db/migrations/20221124191002_deals_add_fast_retrieval.sql

@@ -0,0 +1,10 @@
+-- +goose Up
+-- +goose StatementBegin
+ALTER TABLE Deals
+    ADD FastRetrieval BOOL;
+-- +goose StatementEnd
+
+-- +goose Down
+-- +goose StatementBegin
+SELECT 'down SQL query';
+-- +goose StatementEnd

db/migrations/20221124191256_deals_set_fast_retrieval.go

@@ -0,0 +1,24 @@
+package migrations
+
+import (
+	"database/sql"
+
+	"github.com/pressly/goose/v3"
+)
+
+func init() {
+	goose.AddMigration(upSetdealsfastretrieval, downSetdealsfastretrieval)
+}
+
+func upSetdealsfastretrieval(tx *sql.Tx) error {
+	_, err := tx.Exec("UPDATE Deals SET FastRetrieval=?;", true)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func downSetdealsfastretrieval(tx *sql.Tx) error {
+	// This code is executed when the migration is rolled back.
+	return nil
+}

db/migrations_tests/deals_fast_retrieval_test.go

@@ -0,0 +1,46 @@
+package migrations_tests
+
+import (
+	"context"
+	"testing"
+
+	"github.com/filecoin-project/boost/db"
+	"github.com/filecoin-project/boost/db/migrations"
+	"github.com/pressly/goose/v3"
+	"github.com/stretchr/testify/require"
+)
+
+func TestDealFastRetrieval(t *testing.T) {
+	req := require.New(t)
+	ctx := context.Background()
+
+	sqldb := db.CreateTestTmpDB(t)
+	req.NoError(db.CreateAllBoostTables(ctx, sqldb, sqldb))
+
+	// Run migrations up to the one that adds the FastRetrieval field to Deals
+	goose.SetBaseFS(migrations.EmbedMigrations)
+	req.NoError(goose.SetDialect("sqlite3"))
+	req.NoError(goose.UpTo(sqldb, ".", 20221124191002))
+
+	// Generate 2 deals
+	dealsDB := db.NewDealsDB(sqldb)
+	deals, err := db.GenerateNDeals(1)
+	req.NoError(err)
+
+	// Insert the deals in DB
+	err = dealsDB.Insert(ctx, &deals[0])
+	require.NoError(t, err)
+
+	// Get deal state
+	dealState, err := dealsDB.ByID(ctx, deals[0].DealUuid)
+	require.NoError(t, err)
+	require.False(t, dealState.FastRetrieval)
+
+	//Run migration
+	req.NoError(goose.UpByOne(sqldb, "."))
+
+	// Check the deal state again
+	dealState, err = dealsDB.ByID(ctx, deals[0].DealUuid)
+	require.NoError(t, err)
+	require.True(t, dealState.FastRetrieval)
+}

db/migrations_tests/storagetagged_set_host_test.go

@@ -3,12 +3,13 @@ package migrations_tests
 import (
 	"context"
 	"fmt"
+	"testing"
+
 	"github.com/filecoin-project/boost/db"
 	"github.com/filecoin-project/boost/db/migrations"
 	"github.com/filecoin-project/boost/storagemarket/types"
 	"github.com/pressly/goose/v3"
 	"github.com/stretchr/testify/require"
-	"testing"
 )
 
 func TestStorageTaggedSetHost(t *testing.T) {
@@ -25,6 +26,11 @@ func TestStorageTaggedSetHost(t *testing.T) {
 
 	// Generate 2 deals
 	dealsDB := db.NewDealsDB(sqldb)
+
+	// Add FastRetrieval to allow tests to works
+	_, err := sqldb.Exec(`ALTER TABLE Deals ADD FastRetrieval BOOL`)
+	require.NoError(t, err)
+
 	deals, err := db.GenerateNDeals(2)
 	req.NoError(err)
 

documentation/en/api-v1-methods.md

@@ -397,7 +397,8 @@ Response:
   "CheckpointAt": "0001-01-01T00:00:00Z",
   "Err": "string value",
   "Retry": "auto",
-  "NBytesReceived": 9
+  "NBytesReceived": 9,
+  "FastRetrieval": true
 }
 ```
 
@@ -462,7 +463,8 @@ Response:
   "CheckpointAt": "0001-01-01T00:00:00Z",
   "Err": "string value",
   "Retry": "auto",
-  "NBytesReceived": 9
+  "NBytesReceived": 9,
+  "FastRetrieval": true
 }
 ```
 
@@ -506,7 +508,8 @@ Inputs:
       "ClientID": "string value",
       "Params": "Ynl0ZSBhcnJheQ==",
       "Size": 42
-    }
+    },
+    "FastRetrieval": true
   }
 ]
 ```

indexprovider/wrapper.go

@@ -186,7 +186,7 @@ func (w *Wrapper) AnnounceBoostDeal(ctx context.Context, pds *types.ProviderDeal
 	protocols := []metadata.Protocol{
 		&metadata.GraphsyncFilecoinV1{
 			PieceCID:      pds.ClientDealProposal.Proposal.PieceCID,
-			FastRetrieval: true,
+			FastRetrieval: pds.FastRetrieval,
 			VerifiedDeal:  pds.ClientDealProposal.Proposal.VerifiedDeal,
 		},
 	}

itests/framework/framework.go

@@ -541,6 +541,7 @@ func (f *TestFramework) MakeDummyDeal(dealUuid uuid.UUID, carFilepath string, ro
 			Params: transferParamsJSON,
 			Size:   uint64(carFileinfo.Size()),
 		},
+		FastRetrieval: true,
 	}
 
 	return f.Client.StorageDeal(f.ctx, dealParams, peerID)

storagemarket/provider.go

@@ -277,6 +277,7 @@ func (p *Provider) ExecuteDeal(ctx context.Context, dp *types.DealParams, client
 		Transfer:           dp.Transfer,
 		IsOffline:          dp.IsOffline,
 		Retry:              smtypes.DealRetryAuto,
+		FastRetrieval:      dp.FastRetrieval,
 	}
 	// validate the deal proposal
 	if err := p.validateDealProposal(ds); err != nil {
@@ -586,10 +587,7 @@ func (p *Provider) AddPieceToSector(ctx context.Context, deal smtypes.ProviderDe
 			StartEpoch: deal.ClientDealProposal.Proposal.StartEpoch,
 			EndEpoch:   deal.ClientDealProposal.Proposal.EndEpoch,
 		},
-		// Assume that it doesn't make sense for a miner not to keep an
-		// unsealed copy. TODO: Check that's a valid assumption.
-		//KeepUnsealed: deal.FastRetrieval,
-		KeepUnsealed: true,
+		KeepUnsealed: deal.FastRetrieval,
 	}
 
 	// Attempt to add the piece to a sector (repeatedly if necessary)

storagemarket/provider_test.go

@@ -627,6 +627,10 @@ func TestDealAutoRestartAfterAutoRecoverableErrors(t *testing.T) {
 			require.NoError(t, err)
 			dh := harness.Provider.getDealHandler(td.params.DealUUID)
 			require.NotNil(t, dh)
+
+			//Check for fast retrieval
+			require.False(t, td.params.FastRetrieval)
+
 			sub, err := dh.subscribeUpdates()
 			require.NoError(t, err)
 			td.sub = sub
@@ -1792,6 +1796,7 @@ func (ph *ProviderHarness) newDealBuilder(t *testing.T, seed int, opts ...dealPr
 			Params: xferParams,
 			Size:   uint64(carv2Fileinfo.Size()),
 		},
+		FastRetrieval: false,
 	}
 
 	td := &testDeal{

storagemarket/smtestutil/mocks.go

@@ -262,7 +262,7 @@ func (mb *MinerStubBuilder) SetupAddPiece(blocking bool) *MinerStubBuilder {
 			StartEpoch: mb.dp.ClientDealProposal.Proposal.StartEpoch,
 			EndEpoch:   mb.dp.ClientDealProposal.Proposal.EndEpoch,
 		},
-		KeepUnsealed: true,
+		KeepUnsealed: mb.dp.FastRetrieval,
 	}
 
 	var readBytes []byte
@@ -300,7 +300,7 @@ func (mb *MinerStubBuilder) SetupAddPieceFailure(err error) {
 			StartEpoch: mb.dp.ClientDealProposal.Proposal.StartEpoch,
 			EndEpoch:   mb.dp.ClientDealProposal.Proposal.EndEpoch,
 		},
-		KeepUnsealed: true,
+		KeepUnsealed: mb.dp.FastRetrieval,
 	}
 
 	mb.stub.MockPieceAdder.EXPECT().AddPiece(gomock.Any(), gomock.Eq(mb.dp.ClientDealProposal.Proposal.PieceSize.Unpadded()), gomock.Any(), gomock.Eq(sdInfo)).DoAndReturn(func(_ context.Context, _ abi.UnpaddedPieceSize, r io.Reader, _ api.PieceDealInfo) (abi.SectorNumber, abi.PaddedPieceSize, error) {

storagemarket/types/deal_state.go

@@ -58,6 +58,9 @@ type ProviderDealState struct {
 
 	// NBytesReceived is the number of bytes Received for this deal
 	NBytesReceived int64
+
+	// Keep unsealed copy of the data
+	FastRetrieval bool
 }
 
 func (d *ProviderDealState) String() string {

storagemarket/types/types.go

@@ -83,6 +83,7 @@ type DealParams struct {
 	ClientDealProposal market.ClientDealProposal
 	DealDataRoot       cid.Cid
 	Transfer           Transfer // Transfer params will be the zero value if this is an offline deal
+	FastRetrieval      bool
 }
 
 type DealFilterParams struct {