From 4326040accb0eeb8e060e5d553e8c2bc118ccbf0 Mon Sep 17 00:00:00 2001 From: frrist Date: Fri, 6 May 2022 12:35:22 -0400 Subject: [PATCH] deps: update go-amt-ipld w/ parallel diffing --- chain/actors/adt/diff/array.go | 22 ++++++++++++++++++++-- chain/actors/builtin/market/diff.go | 2 +- chain/actors/builtin/miner/diff.go | 2 +- go.mod | 4 ++-- go.sum | 3 ++- 5 files changed, 26 insertions(+), 7 deletions(-) diff --git a/chain/actors/adt/diff/array.go b/chain/actors/adt/diff/array.go index ab15ad004..222eeca13 100644 --- a/chain/actors/adt/diff/array.go +++ b/chain/actors/adt/diff/array.go @@ -2,14 +2,32 @@ package diff import ( "context" + "os" + "runtime" + "strconv" - "github.com/filecoin-project/go-amt-ipld/v3" + "github.com/filecoin-project/go-amt-ipld/v4" "github.com/filecoin-project/lotus/chain/actors/adt" "go.opentelemetry.io/otel" adt2 "github.com/filecoin-project/lily/chain/actors/adt" ) +var AmtParallelWorkerLimit int64 + +const AmtParallelWorkerEnv = "LILY_AMT_PARALLEL_WORKER_LIMIT" + +func init() { + AmtParallelWorkerLimit = int64(runtime.NumCPU()) + workerStr := os.Getenv(AmtParallelWorkerEnv) + if workerStr != "" { + AmtParallelWorkerLimit, err := strconv.ParseInt(workerStr, 10, 64) + if err != nil { + log.Warnf("failed to parse env %s defaulting to %d (runtime.NumCPU()) : %s", AmtParallelWorkerEnv, AmtParallelWorkerLimit, err) + } + } +} + // Amt returns a set of changes that transform `preArr` into `curArr`. opts are applied to both `preArr` and `curArr`. func Amt(ctx context.Context, preArr, curArr adt2.Array, preStore, curStore adt.Store, amtOpts ...amt.Option) ([]*amt.Change, error) { ctx, span := otel.Tracer("").Start(ctx, "Amt.Diff") @@ -25,5 +43,5 @@ func Amt(ctx context.Context, preArr, curArr adt2.Array, preStore, curStore adt. return nil, err } - return amt.Diff(ctx, preStore, curStore, preRoot, curRoot, amtOpts...) + return amt.ParallelDiff(ctx, preStore, curStore, preRoot, curRoot, AmtParallelWorkerLimit, amtOpts...) } diff --git a/chain/actors/builtin/market/diff.go b/chain/actors/builtin/market/diff.go index 5870552b5..283e29d09 100644 --- a/chain/actors/builtin/market/diff.go +++ b/chain/actors/builtin/market/diff.go @@ -7,7 +7,7 @@ import ( logging "github.com/ipfs/go-log/v2" cbg "github.com/whyrusleeping/cbor-gen" - "github.com/filecoin-project/go-amt-ipld/v3" + "github.com/filecoin-project/go-amt-ipld/v4" "github.com/filecoin-project/go-state-types/abi" builtin0 "github.com/filecoin-project/specs-actors/actors/builtin" diff --git a/chain/actors/builtin/miner/diff.go b/chain/actors/builtin/miner/diff.go index c0e604cfd..91e73e77e 100644 --- a/chain/actors/builtin/miner/diff.go +++ b/chain/actors/builtin/miner/diff.go @@ -8,7 +8,7 @@ import ( "go.opentelemetry.io/otel/attribute" "golang.org/x/xerrors" - "github.com/filecoin-project/go-amt-ipld/v3" + "github.com/filecoin-project/go-amt-ipld/v4" "github.com/filecoin-project/go-hamt-ipld/v3" "github.com/filecoin-project/go-state-types/abi" diff --git a/go.mod b/go.mod index 6008af5fa..8acee9cc2 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/OneOfOne/xxhash v1.2.8 // indirect github.com/filecoin-project/go-address v0.0.6 github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20201006184820-924ee87a1349 // indirect - github.com/filecoin-project/go-amt-ipld/v3 v3.1.1 + github.com/filecoin-project/go-amt-ipld/v3 v3.1.1 // indirect github.com/filecoin-project/go-bitfield v0.2.4 github.com/filecoin-project/go-hamt-ipld/v3 v3.1.1-0.20220505191157-d7766f8628ec github.com/filecoin-project/go-jsonrpc v0.1.5 @@ -63,6 +63,7 @@ require ( require k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9 require ( + github.com/filecoin-project/go-amt-ipld/v4 v4.0.1-0.20220506152917-1e3e6c61ff37 github.com/hibiken/asynq v0.23.0 github.com/hibiken/asynq/x v0.0.0-20220413130846-5c723f597e01 github.com/jedib0t/go-pretty/v6 v6.2.7 @@ -114,7 +115,6 @@ require ( github.com/fatih/color v1.13.0 // indirect github.com/filecoin-project/dagstore v0.5.2 // indirect github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f // indirect - github.com/filecoin-project/go-amt-ipld/v4 v4.0.0 // indirect github.com/filecoin-project/go-cbor-util v0.0.1 // indirect github.com/filecoin-project/go-commp-utils v0.1.3 // indirect github.com/filecoin-project/go-crypto v0.0.1 // indirect diff --git a/go.sum b/go.sum index 8ba122958..86b902ec2 100644 --- a/go.sum +++ b/go.sum @@ -309,8 +309,9 @@ github.com/filecoin-project/go-amt-ipld/v3 v3.0.0/go.mod h1:Qa95YNAbtoVCTSVtX38a github.com/filecoin-project/go-amt-ipld/v3 v3.1.0/go.mod h1:UjM2QhDFrrjD5s1CdnkJkat4ga+LqZBZgTMniypABRo= github.com/filecoin-project/go-amt-ipld/v3 v3.1.1 h1:n+nczYe6VedXmdtAXygRuey246YnYyuY1NPrmy2iK6s= github.com/filecoin-project/go-amt-ipld/v3 v3.1.1/go.mod h1:UjM2QhDFrrjD5s1CdnkJkat4ga+LqZBZgTMniypABRo= -github.com/filecoin-project/go-amt-ipld/v4 v4.0.0 h1:XM81BJ4/6h3FV0WfFjh74cIDIgqMbJsMBLM0fIuLUUk= github.com/filecoin-project/go-amt-ipld/v4 v4.0.0/go.mod h1:gF053YQ4BIpzTNDoEwHZas7U3oAwncDVGvOHyY8oDpE= +github.com/filecoin-project/go-amt-ipld/v4 v4.0.1-0.20220506152917-1e3e6c61ff37 h1:TV+fo0W8NVZS3brA4lbcgi8qhPcvfb6B6sfW2kXft5s= +github.com/filecoin-project/go-amt-ipld/v4 v4.0.1-0.20220506152917-1e3e6c61ff37/go.mod h1:I2C4qb5X5qNqcguwvA95ycNqRe3NSc6iD3EuzBfelPU= github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-bitfield v0.2.3/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM= github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW8p9au0C68JPgk=