Skip to content

deps: update go-amt-ipld w/ parallel diffing #953

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
May 6, 2022
Merged

deps: update go-amt-ipld w/ parallel diffing #953

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 20 additions & 2 deletions chain/actors/adt/diff/array.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,14 +2,32 @@ package diff

import (
"context"
"os"
"runtime"
"strconv"

"github.com/filecoin-project/go-amt-ipld/v3"
"github.com/filecoin-project/go-amt-ipld/v4"
"github.com/filecoin-project/lotus/chain/actors/adt"
"go.opentelemetry.io/otel"

adt2 "github.com/filecoin-project/lily/chain/actors/adt"
)

var AmtParallelWorkerLimit int64

const AmtParallelWorkerEnv = "LILY_AMT_PARALLEL_WORKER_LIMIT"

func init() {
AmtParallelWorkerLimit = int64(runtime.NumCPU())
workerStr := os.Getenv(AmtParallelWorkerEnv)
if workerStr != "" {
AmtParallelWorkerLimit, err := strconv.ParseInt(workerStr, 10, 64)
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: Put this error check in the if above to flatten.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Actually, that would ruin the AmtParallelWorkerLimit on error. nm. GTM

if err != nil {
log.Warnf("failed to parse env %s defaulting to %d (runtime.NumCPU()) : %s", AmtParallelWorkerEnv, AmtParallelWorkerLimit, err)
}
}
}

// Amt returns a set of changes that transform `preArr` into `curArr`. opts are applied to both `preArr` and `curArr`.
func Amt(ctx context.Context, preArr, curArr adt2.Array, preStore, curStore adt.Store, amtOpts ...amt.Option) ([]*amt.Change, error) {
ctx, span := otel.Tracer("").Start(ctx, "Amt.Diff")
Expand All @@ -25,5 +43,5 @@ func Amt(ctx context.Context, preArr, curArr adt2.Array, preStore, curStore adt.
return nil, err
}

return amt.Diff(ctx, preStore, curStore, preRoot, curRoot, amtOpts...)
return amt.ParallelDiff(ctx, preStore, curStore, preRoot, curRoot, AmtParallelWorkerLimit, amtOpts...)
}
2 changes: 1 addition & 1 deletion chain/actors/builtin/market/diff.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import (
logging "github.com/ipfs/go-log/v2"
cbg "github.com/whyrusleeping/cbor-gen"

"github.com/filecoin-project/go-amt-ipld/v3"
"github.com/filecoin-project/go-amt-ipld/v4"
"github.com/filecoin-project/go-state-types/abi"

builtin0 "github.com/filecoin-project/specs-actors/actors/builtin"
Expand Down
2 changes: 1 addition & 1 deletion chain/actors/builtin/miner/diff.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ import (
"go.opentelemetry.io/otel/attribute"
"golang.org/x/xerrors"

"github.com/filecoin-project/go-amt-ipld/v3"
"github.com/filecoin-project/go-amt-ipld/v4"
"github.com/filecoin-project/go-hamt-ipld/v3"
"github.com/filecoin-project/go-state-types/abi"

Expand Down
4 changes: 2 additions & 2 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@ require (
github.com/OneOfOne/xxhash v1.2.8 // indirect
github.com/filecoin-project/go-address v0.0.6
github.com/filecoin-project/go-amt-ipld/v2 v2.1.1-0.20201006184820-924ee87a1349 // indirect
github.com/filecoin-project/go-amt-ipld/v3 v3.1.1
github.com/filecoin-project/go-amt-ipld/v3 v3.1.1 // indirect
github.com/filecoin-project/go-bitfield v0.2.4
github.com/filecoin-project/go-hamt-ipld/v3 v3.1.1-0.20220505191157-d7766f8628ec
github.com/filecoin-project/go-jsonrpc v0.1.5
Expand Down Expand Up @@ -63,6 +63,7 @@ require (
require k8s.io/utils v0.0.0-20220210201930-3a6ce19ff2f9

require (
github.com/filecoin-project/go-amt-ipld/v4 v4.0.1-0.20220506152917-1e3e6c61ff37
github.com/hibiken/asynq v0.23.0
github.com/hibiken/asynq/x v0.0.0-20220413130846-5c723f597e01
github.com/jedib0t/go-pretty/v6 v6.2.7
Expand Down Expand Up @@ -114,7 +115,6 @@ require (
github.com/fatih/color v1.13.0 // indirect
github.com/filecoin-project/dagstore v0.5.2 // indirect
github.com/filecoin-project/filecoin-ffi v0.30.4-0.20200910194244-f640612a1a1f // indirect
github.com/filecoin-project/go-amt-ipld/v4 v4.0.0 // indirect
github.com/filecoin-project/go-cbor-util v0.0.1 // indirect
github.com/filecoin-project/go-commp-utils v0.1.3 // indirect
github.com/filecoin-project/go-crypto v0.0.1 // indirect
Expand Down
3 changes: 2 additions & 1 deletion go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -309,8 +309,9 @@ github.com/filecoin-project/go-amt-ipld/v3 v3.0.0/go.mod h1:Qa95YNAbtoVCTSVtX38a
github.com/filecoin-project/go-amt-ipld/v3 v3.1.0/go.mod h1:UjM2QhDFrrjD5s1CdnkJkat4ga+LqZBZgTMniypABRo=
github.com/filecoin-project/go-amt-ipld/v3 v3.1.1 h1:n+nczYe6VedXmdtAXygRuey246YnYyuY1NPrmy2iK6s=
github.com/filecoin-project/go-amt-ipld/v3 v3.1.1/go.mod h1:UjM2QhDFrrjD5s1CdnkJkat4ga+LqZBZgTMniypABRo=
github.com/filecoin-project/go-amt-ipld/v4 v4.0.0 h1:XM81BJ4/6h3FV0WfFjh74cIDIgqMbJsMBLM0fIuLUUk=
github.com/filecoin-project/go-amt-ipld/v4 v4.0.0/go.mod h1:gF053YQ4BIpzTNDoEwHZas7U3oAwncDVGvOHyY8oDpE=
github.com/filecoin-project/go-amt-ipld/v4 v4.0.1-0.20220506152917-1e3e6c61ff37 h1:TV+fo0W8NVZS3brA4lbcgi8qhPcvfb6B6sfW2kXft5s=
github.com/filecoin-project/go-amt-ipld/v4 v4.0.1-0.20220506152917-1e3e6c61ff37/go.mod h1:I2C4qb5X5qNqcguwvA95ycNqRe3NSc6iD3EuzBfelPU=
github.com/filecoin-project/go-bitfield v0.2.0/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
github.com/filecoin-project/go-bitfield v0.2.3/go.mod h1:CNl9WG8hgR5mttCnUErjcQjGvuiZjRqK9rHVBsQF4oM=
github.com/filecoin-project/go-bitfield v0.2.4 h1:uZ7MeE+XfM5lqrHJZ93OnhQKc/rveW8p9au0C68JPgk=
Expand Down