/
storage.go
217 lines (168 loc) · 7.41 KB
/
storage.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
package storiface
import (
"context"
"io"
"net/http"
"github.com/ipfs/go-cid"
"github.com/filecoin-project/go-state-types/abi"
"github.com/filecoin-project/go-state-types/proof"
)
type Data = io.Reader
type SectorRef struct {
ID abi.SectorID
ProofType abi.RegisteredSealProof
}
var NoSectorRef = SectorRef{}
type ProverPoSt interface {
GenerateWinningPoSt(ctx context.Context, minerID abi.ActorID, sectorInfo []proof.ExtendedSectorInfo, randomness abi.PoStRandomness) ([]proof.PoStProof, error)
GenerateWindowPoSt(ctx context.Context, minerID abi.ActorID, ppt abi.RegisteredPoStProof, sectorInfo []proof.ExtendedSectorInfo, randomness abi.PoStRandomness) (proof []proof.PoStProof, skipped []abi.SectorID, err error)
GenerateWinningPoStWithVanilla(ctx context.Context, proofType abi.RegisteredPoStProof, minerID abi.ActorID, randomness abi.PoStRandomness, proofs [][]byte) ([]proof.PoStProof, error)
GenerateWindowPoStWithVanilla(ctx context.Context, proofType abi.RegisteredPoStProof, minerID abi.ActorID, randomness abi.PoStRandomness, proofs [][]byte, partitionIdx int) (proof.PoStProof, error)
}
type PreCommit1Out []byte
type Commit1Out []byte
type Proof []byte
type SectorCids struct {
Unsealed cid.Cid
Sealed cid.Cid
}
type Range struct {
Offset abi.UnpaddedPieceSize
Size abi.UnpaddedPieceSize
}
type ReplicaUpdateProof []byte
type ReplicaVanillaProofs [][]byte
type ReplicaUpdateOut struct {
NewSealed cid.Cid
NewUnsealed cid.Cid
}
type Sealer interface {
NewSector(ctx context.Context, sector SectorRef) error
DataCid(ctx context.Context, pieceSize abi.UnpaddedPieceSize, pieceData Data) (abi.PieceInfo, error)
AddPiece(ctx context.Context, sector SectorRef, pieceSizes []abi.UnpaddedPieceSize, newPieceSize abi.UnpaddedPieceSize, pieceData Data) (abi.PieceInfo, error)
SealPreCommit1(ctx context.Context, sector SectorRef, ticket abi.SealRandomness, pieces []abi.PieceInfo) (PreCommit1Out, error)
SealPreCommit2(ctx context.Context, sector SectorRef, pc1o PreCommit1Out) (SectorCids, error)
SealCommit1(ctx context.Context, sector SectorRef, ticket abi.SealRandomness, seed abi.InteractiveSealRandomness, pieces []abi.PieceInfo, cids SectorCids) (Commit1Out, error)
SealCommit2(ctx context.Context, sector SectorRef, c1o Commit1Out) (Proof, error)
FinalizeSector(ctx context.Context, sector SectorRef) error
// ReleaseUnsealed marks parts of the unsealed sector file as safe to drop
// (called by the fsm on restart, allows storage to keep no persistent
// state about unsealed fast-retrieval copies)
ReleaseUnsealed(ctx context.Context, sector SectorRef, keepUnsealed []Range) error
// ReleaseSectorKey removes `sealed` from all storage
// called after successful sector upgrade
ReleaseSectorKey(ctx context.Context, sector SectorRef) error
// ReleaseReplicaUpgrade removes `update` / `update-cache` from all storage
// called when aborting sector upgrade
ReleaseReplicaUpgrade(ctx context.Context, sector SectorRef) error
// Removes all data associated with the specified sector
Remove(ctx context.Context, sector SectorRef) error
// Generate snap deals replica update
ReplicaUpdate(ctx context.Context, sector SectorRef, pieces []abi.PieceInfo) (ReplicaUpdateOut, error)
// Prove that snap deals replica was done correctly
ProveReplicaUpdate1(ctx context.Context, sector SectorRef, sectorKey, newSealed, newUnsealed cid.Cid) (ReplicaVanillaProofs, error)
ProveReplicaUpdate2(ctx context.Context, sector SectorRef, sectorKey, newSealed, newUnsealed cid.Cid, vanillaProofs ReplicaVanillaProofs) (ReplicaUpdateProof, error)
// GenerateSectorKeyFromData computes sector key given unsealed data and updated replica
GenerateSectorKeyFromData(ctx context.Context, sector SectorRef, unsealed cid.Cid) error
FinalizeReplicaUpdate(ctx context.Context, sector SectorRef) error
DownloadSectorData(ctx context.Context, sector SectorRef, finalized bool, src map[SectorFileType]SectorLocation) error
}
type Unsealer interface {
UnsealPiece(ctx context.Context, sector SectorRef, offset UnpaddedByteIndex, size abi.UnpaddedPieceSize, randomness abi.SealRandomness, commd cid.Cid) error
ReadPiece(ctx context.Context, writer io.Writer, sector SectorRef, offset UnpaddedByteIndex, size abi.UnpaddedPieceSize) (bool, error)
}
type Storage interface {
ProverPoSt
Sealer
Unsealer
}
type Validator interface {
CanCommit(sector SectorPaths) (bool, error)
CanProve(sector SectorPaths) (bool, error)
}
type Verifier interface {
VerifySeal(proof.SealVerifyInfo) (bool, error)
VerifyAggregateSeals(aggregate proof.AggregateSealVerifyProofAndInfos) (bool, error)
VerifyReplicaUpdate(update proof.ReplicaUpdateInfo) (bool, error)
VerifyWinningPoSt(ctx context.Context, info proof.WinningPoStVerifyInfo) (bool, error)
VerifyWindowPoSt(ctx context.Context, info proof.WindowPoStVerifyInfo) (bool, error)
GenerateWinningPoStSectorChallenge(context.Context, abi.RegisteredPoStProof, abi.ActorID, abi.PoStRandomness, uint64) ([]uint64, error)
}
// Prover contains cheap proving-related methods
type Prover interface {
// TODO: move GenerateWinningPoStSectorChallenge from the Verifier interface to here
AggregateSealProofs(aggregateInfo proof.AggregateSealVerifyProofAndInfos, proofs [][]byte) ([]byte, error)
}
type SectorLocation struct {
// Local when set to true indicates to lotus that sector data is already
// available locally; When set lotus will skip fetching sector data, and
// only check that sector data exists in sector storage
Local bool
// URL to the sector data
// For sealed/unsealed sector, lotus expects octet-stream
// For cache, lotus expects a tar archive with cache files
// Valid schemas:
// - http:// / https://
URL string
// optional http headers to use when requesting sector data
Headers []SecDataHttpHeader
}
func (sd *SectorLocation) HttpHeaders() http.Header {
out := http.Header{}
for _, header := range sd.Headers {
out[header.Key] = append(out[header.Key], header.Value)
}
return out
}
// note: we can't use http.Header as that's backed by a go map, which is all kinds of messy
type SecDataHttpHeader struct {
Key string
Value string
}
// StorageConfig .lotusstorage/storage.json
type StorageConfig struct {
StoragePaths []LocalPath
}
type LocalPath struct {
Path string
}
// LocalStorageMeta [path]/sectorstore.json
type LocalStorageMeta struct {
ID ID
// A high weight means data is more likely to be stored in this path
Weight uint64 // 0 = readonly
// Intermediate data for the sealing process will be stored here
CanSeal bool
// Finalized sectors that will be proved over time will be stored here
CanStore bool
// MaxStorage specifies the maximum number of bytes to use for sector storage
// (0 = unlimited)
MaxStorage uint64
// List of storage groups this path belongs to
Groups []string
// List of storage groups to which data from this path can be moved. If none
// are specified, allow to all
AllowTo []string
// AllowTypes lists sector file types which are allowed to be put into this
// path. If empty, all file types are allowed.
//
// Valid values:
// - "unsealed"
// - "sealed"
// - "cache"
// - "update"
// - "update-cache"
// Any other value will generate a warning and be ignored.
AllowTypes []string
// DenyTypes lists sector file types which aren't allowed to be put into this
// path.
//
// Valid values:
// - "unsealed"
// - "sealed"
// - "cache"
// - "update"
// - "update-cache"
// Any other value will generate a warning and be ignored.
DenyTypes []string
}