Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
e1cbea7
feat: implement DataSource API
frrist Feb 9, 2022
924aaab
refactor: simplify indexer processors
frrist Feb 9, 2022
4631f77
refactor: ActorStateChangeDiff contains addresses, add TipSet method …
frrist Feb 16, 2022
428899b
polish: add tracing to adt diff method
frrist Feb 22, 2022
58b5090
refactor: export miner actor diffing logic
frrist Feb 22, 2022
a527b23
refactor: DataSource API
frrist Feb 22, 2022
e5ac446
refactor: break up actorstate tasks per model extracted
frrist Feb 22, 2022
e805b5d
refactor: break up message tasks per model extracted
frrist Feb 22, 2022
bf38799
refactor: block, msapprovals, economics, tasks consensus
frrist Feb 22, 2022
ef1832f
refactor: imlement ReportProcessor task
frrist Feb 22, 2022
eec896f
refactor: rename GetExecutedAndBlockMessagesForTipset params
frrist Feb 22, 2022
299501b
refactor: chain indexer
frrist Feb 22, 2022
104b84c
refactor: update walk, watch, fill, and find to work with new indexer
frrist Feb 22, 2022
8aeb23f
refactor: update itests to work with new indexer tasks
frrist Feb 22, 2022
12c5c4d
feat: implement index command
frrist Feb 22, 2022
2018178
cleanup: remove unused structs and TODO comment
frrist Feb 22, 2022
b8e703e
refactor: break up internal message task per model extracte
frrist Feb 24, 2022
b1b2c0d
refactor: add miner sector info v7 task
frrist Feb 24, 2022
86f91c6
refactor: remove dead code
frrist Feb 24, 2022
c8954d6
refactor: replace general tasks with model specific tasks
frrist Feb 24, 2022
42ed36b
refactor: record metrics
frrist Feb 24, 2022
0443b63
cleanup: fix lint
frrist Feb 24, 2022
5aeeed0
fix: don't initialize zero value singleflight.Group
frrist Mar 2, 2022
09b3760
polish: add metrics and env setters for caches
frrist Mar 2, 2022
dc1b43a
polish: use address and tipsets for miner diffing keys
frrist Mar 2, 2022
7fa0767
cleanup: address nits on manager construction
frrist Mar 2, 2022
a125fc8
fix: manager locking for fatal error
frrist Mar 2, 2022
476f78f
cleanup: state processor logs and interface comments
frrist Mar 2, 2022
f6c95d3
fix: GetExecutedAndBlockMessagesForTipset error message
frrist Mar 2, 2022
ac9fa2d
fix: remove unused miner state extractor
frrist Mar 2, 2022
7863a3d
cleanup: remove market state extractor
frrist Mar 3, 2022
94e8068
clanup: close backtick on code comment
frrist Mar 3, 2022
e88c904
cleanup: parseTipSetKey accepts a string
frrist Mar 3, 2022
76dcac8
refactor: rename indexer to index
frrist Mar 4, 2022
42c3bb8
feat: implement distributed worker pattern using asynq
frrist Mar 4, 2022
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 6 additions & 1 deletion chain/actors/adt/diff/array.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,17 @@ import (
"context"

"github.com/filecoin-project/go-amt-ipld/v3"
adt2 "github.com/filecoin-project/lily/chain/actors/adt"
"github.com/filecoin-project/lotus/chain/actors/adt"
"go.opentelemetry.io/otel"

adt2 "github.com/filecoin-project/lily/chain/actors/adt"
)

// Amt returns a set of changes that transform `preArr` into `curArr`. opts are applied to both `preArr` and `curArr`.
func Amt(ctx context.Context, preArr, curArr adt2.Array, preStore, curStore adt.Store, amtOpts ...amt.Option) ([]*amt.Change, error) {
ctx, span := otel.Tracer("").Start(ctx, "Amt.Diff")
defer span.End()

preRoot, err := preArr.Root()
if err != nil {
return nil, err
Expand Down
7 changes: 6 additions & 1 deletion chain/actors/adt/diff/map.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,17 @@ import (
"context"

"github.com/filecoin-project/go-hamt-ipld/v3"
adt2 "github.com/filecoin-project/lily/chain/actors/adt"
"github.com/filecoin-project/lotus/chain/actors/adt"
"go.opentelemetry.io/otel"

adt2 "github.com/filecoin-project/lily/chain/actors/adt"
)

// Hamt returns a set of changes that transform `preMap` into `curMap`. opts are applied to both `preMap` and `curMap`.
func Hamt(ctx context.Context, preMap, curMap adt2.Map, preStore, curStore adt.Store, hamtOpts ...hamt.Option) ([]*hamt.Change, error) {
ctx, span := otel.Tracer("").Start(ctx, "Hamt.Diff")
defer span.End()

preRoot, err := preMap.Root()
if err != nil {
return nil, err
Expand Down
11 changes: 6 additions & 5 deletions chain/actors/builtin/miner/actor.go.template
Original file line number Diff line number Diff line change
Expand Up @@ -113,12 +113,13 @@ type State interface {

SectorsAmtBitwidth() int

// Diff helpers. Used by Diff* functions internally.
sectors() (adt.Array, error)
decodeSectorOnChainInfo(*cbg.Deferred) (SectorOnChainInfo, error)
precommits() (adt.Map, error)
decodeSectorPreCommitOnChainInfo(*cbg.Deferred) (SectorPreCommitOnChainInfo, error)
GetState() interface{}

// Diff helpers. Used by Diff* functions internally.
SectorArray() (adt.Array, error)
DecodeSectorOnChainInfo(*cbg.Deferred) (SectorOnChainInfo, error)
PreCommitMap() (adt.Map, error)
DecodeSectorPreCommitOnChainInfo(*cbg.Deferred) (SectorPreCommitOnChainInfo, error)
}

type Deadline interface {
Expand Down
28 changes: 14 additions & 14 deletions chain/actors/builtin/miner/diff.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,12 @@ import (
func DiffPreCommits(ctx context.Context, store adt.Store, pre, cur State) (*PreCommitChanges, error) {
ctx, span := otel.Tracer("").Start(ctx, "DiffPreCommits")
defer span.End()
prep, err := pre.precommits()
prep, err := pre.PreCommitMap()
if err != nil {
return nil, err
}

curp, err := cur.precommits()
curp, err := cur.PreCommitMap()
if err != nil {
return nil, err
}
Expand All @@ -42,7 +42,7 @@ func DiffPreCommits(ctx context.Context, store adt.Store, pre, cur State) (*PreC
}

diffContainer := NewPreCommitDiffContainer(pre, cur)
if mapRequiresLegacyDiffing(pre, cur, preOpts, curOpts) {
if MapRequiresLegacyDiffing(pre, cur, preOpts, curOpts) {
if span.IsRecording() {
span.SetAttributes(attribute.String("diff", "slow"))
}
Expand Down Expand Up @@ -102,7 +102,7 @@ func (m *preCommitDiffContainer) AsKey(key string) (abi.Keyer, error) {
}

func (m *preCommitDiffContainer) Add(key string, val *cbg.Deferred) error {
sp, err := m.after.decodeSectorPreCommitOnChainInfo(val)
sp, err := m.after.DecodeSectorPreCommitOnChainInfo(val)
if err != nil {
return xerrors.Errorf("pre commit diff container add: %w", err)
}
Expand All @@ -115,7 +115,7 @@ func (m *preCommitDiffContainer) Modify(key string, from, to *cbg.Deferred) erro
}

func (m *preCommitDiffContainer) Remove(key string, val *cbg.Deferred) error {
sp, err := m.pre.decodeSectorPreCommitOnChainInfo(val)
sp, err := m.pre.DecodeSectorPreCommitOnChainInfo(val)
if err != nil {
return xerrors.Errorf("pre commit diff container remove: %w", err)
}
Expand All @@ -126,20 +126,20 @@ func (m *preCommitDiffContainer) Remove(key string, val *cbg.Deferred) error {
func DiffSectors(ctx context.Context, store adt.Store, pre, cur State) (*SectorChanges, error) {
ctx, span := otel.Tracer("").Start(ctx, "DiffSectors")
defer span.End()
pres, err := pre.sectors()
pres, err := pre.SectorArray()
if err != nil {
return nil, err
}

curs, err := cur.sectors()
curs, err := cur.SectorArray()
if err != nil {
return nil, err
}

preBw := pre.SectorsAmtBitwidth()
curBw := cur.SectorsAmtBitwidth()
diffContainer := NewSectorDiffContainer(pre, cur)
if arrayRequiresLegacyDiffing(pre, cur, preBw, curBw) {
if ArrayRequiresLegacyDiffing(pre, cur, preBw, curBw) {
if span.IsRecording() {
span.SetAttributes(attribute.String("diff", "slow"))
}
Expand Down Expand Up @@ -191,7 +191,7 @@ type sectorDiffContainer struct {
}

func (m *sectorDiffContainer) Add(key uint64, val *cbg.Deferred) error {
si, err := m.after.decodeSectorOnChainInfo(val)
si, err := m.after.DecodeSectorOnChainInfo(val)
if err != nil {
return xerrors.Errorf("sector diff container add: %w", err)
}
Expand All @@ -200,12 +200,12 @@ func (m *sectorDiffContainer) Add(key uint64, val *cbg.Deferred) error {
}

func (m *sectorDiffContainer) Modify(key uint64, from, to *cbg.Deferred) error {
siFrom, err := m.pre.decodeSectorOnChainInfo(from)
siFrom, err := m.pre.DecodeSectorOnChainInfo(from)
if err != nil {
return xerrors.Errorf("sector diff container modify from: %w", err)
}

siTo, err := m.after.decodeSectorOnChainInfo(to)
siTo, err := m.after.DecodeSectorOnChainInfo(to)
if err != nil {
return xerrors.Errorf("sector diff container modify to: %w", err)
}
Expand All @@ -229,15 +229,15 @@ func (m *sectorDiffContainer) Modify(key uint64, from, to *cbg.Deferred) error {
}

func (m *sectorDiffContainer) Remove(key uint64, val *cbg.Deferred) error {
si, err := m.pre.decodeSectorOnChainInfo(val)
si, err := m.pre.DecodeSectorOnChainInfo(val)
if err != nil {
return xerrors.Errorf("sector diff container remove: %w", err)
}
m.Results.Removed = append(m.Results.Removed, si)
return nil
}

func arrayRequiresLegacyDiffing(pre, cur State, pOpts, cOpts int) bool {
func ArrayRequiresLegacyDiffing(pre, cur State, pOpts, cOpts int) bool {
// amt/v3 cannot read amt/v2 nodes. Their Pointers struct has changed cbor marshalers.
if pre.Code().Equals(builtin0.StorageMinerActorCodeID) {
return true
Expand All @@ -258,7 +258,7 @@ func arrayRequiresLegacyDiffing(pre, cur State, pOpts, cOpts int) bool {
return false
}

func mapRequiresLegacyDiffing(pre, cur State, pOpts, cOpts *adt.MapOpts) bool {
func MapRequiresLegacyDiffing(pre, cur State, pOpts, cOpts *adt.MapOpts) bool {
// hamt/v3 cannot read hamt/v2 nodes. Their Pointers struct has changed cbor marshalers.
if pre.Code().Equals(builtin0.StorageMinerActorCodeID) {
return true
Expand Down
11 changes: 6 additions & 5 deletions chain/actors/builtin/miner/miner.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions chain/actors/builtin/miner/state.go.template
Original file line number Diff line number Diff line change
Expand Up @@ -432,11 +432,11 @@ func (s *state{{.v}}) DeadlineCronActive() (bool, error) {
return {{if (ge .v 4)}}s.State.DeadlineCronActive{{else}}true{{end}}, nil{{if (lt .v 4)}} // always active in this version{{end}}
}

func (s *state{{.v}}) sectors() (adt.Array, error) {
func (s *state{{.v}}) SectorArray() (adt.Array, error) {
return adt{{.v}}.AsArray(s.store, s.Sectors{{if (ge .v 3)}}, miner{{.v}}.SectorsAmtBitwidth{{end}})
}

func (s *state{{.v}}) decodeSectorOnChainInfo(val *cbg.Deferred) (SectorOnChainInfo, error) {
func (s *state{{.v}}) DecodeSectorOnChainInfo(val *cbg.Deferred) (SectorOnChainInfo, error) {
var si miner{{.v}}.SectorOnChainInfo
err := si.UnmarshalCBOR(bytes.NewReader(val.Raw))
if err != nil {
Expand All @@ -446,11 +446,11 @@ func (s *state{{.v}}) decodeSectorOnChainInfo(val *cbg.Deferred) (SectorOnChainI
return fromV{{.v}}SectorOnChainInfo(si), nil
}

func (s *state{{.v}}) precommits() (adt.Map, error) {
func (s *state{{.v}}) PreCommitMap() (adt.Map, error) {
return adt{{.v}}.AsMap(s.store, s.PreCommittedSectors{{if (ge .v 3)}}, builtin{{.v}}.DefaultHamtBitwidth{{end}})
}

func (s *state{{.v}}) decodeSectorPreCommitOnChainInfo(val *cbg.Deferred) (SectorPreCommitOnChainInfo, error) {
func (s *state{{.v}}) DecodeSectorPreCommitOnChainInfo(val *cbg.Deferred) (SectorPreCommitOnChainInfo, error) {
var sp miner{{.v}}.SectorPreCommitOnChainInfo
err := sp.UnmarshalCBOR(bytes.NewReader(val.Raw))
if err != nil {
Expand Down
8 changes: 4 additions & 4 deletions chain/actors/builtin/miner/v0.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions chain/actors/builtin/miner/v2.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions chain/actors/builtin/miner/v3.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions chain/actors/builtin/miner/v4.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions chain/actors/builtin/miner/v5.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

8 changes: 4 additions & 4 deletions chain/actors/builtin/miner/v6.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading