Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: go-libp2p-kad-dht with expiration 48h #9491

Closed
wants to merge 6 commits into from
Closed

feat: go-libp2p-kad-dht with expiration 48h #9491

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
e71bf5d
stop using the deprecated go-libp2p-loggables package
marten-seemann Nov 20, 2022
ea9aaad
update go-libp2p to v0.24.0
Jorropo Nov 20, 2022
353aed0
fix: silence staticcheck warning for fx.Extract usage
Jorropo Dec 7, 2022
697c789
fix: replace quic to quic-v1 for webtransport sharness
Jorropo Dec 9, 2022
60ad31a
fix: remove the imports work-around
Jorropo Dec 9, 2022
f31d524
feat: go-libp2p-kad-dht with expiration 48h
lidel Dec 11, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 21 additions & 10 deletions cmd/ipfs/main.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,26 +12,27 @@ import (
"runtime/pprof"
"time"

util "github.com/ipfs/kubo/cmd/ipfs/util"
"github.com/ipfs/kubo/cmd/ipfs/util"
oldcmds "github.com/ipfs/kubo/commands"
core "github.com/ipfs/kubo/core"
"github.com/ipfs/kubo/core"
corecmds "github.com/ipfs/kubo/core/commands"
corehttp "github.com/ipfs/kubo/core/corehttp"
loader "github.com/ipfs/kubo/plugin/loader"
repo "github.com/ipfs/kubo/repo"
fsrepo "github.com/ipfs/kubo/repo/fsrepo"
"github.com/ipfs/kubo/core/corehttp"
"github.com/ipfs/kubo/plugin/loader"
"github.com/ipfs/kubo/repo"
"github.com/ipfs/kubo/repo/fsrepo"
"github.com/ipfs/kubo/tracing"
"go.opentelemetry.io/otel"

cmds "github.com/ipfs/go-ipfs-cmds"
"github.com/ipfs/go-ipfs-cmds/cli"
cmdhttp "github.com/ipfs/go-ipfs-cmds/http"
u "github.com/ipfs/go-ipfs-util"
logging "github.com/ipfs/go-log"
loggables "github.com/libp2p/go-libp2p-loggables"
ma "github.com/multiformats/go-multiaddr"
madns "github.com/multiformats/go-multiaddr-dns"
manet "github.com/multiformats/go-multiaddr/net"

"github.com/google/uuid"
"go.opentelemetry.io/otel"
)

// log is the command logger
Expand Down Expand Up @@ -77,9 +78,19 @@ func printErr(err error) int {
return 1
}

func newUUID(key string) logging.Metadata {
ids := "#UUID-ERROR#"
if id, err := uuid.NewRandom(); err == nil {
ids = id.String()
}
return logging.Metadata{
key: ids,
}
}

func mainRet() (exitCode int) {
rand.Seed(time.Now().UnixNano())
ctx := logging.ContextWithLoggable(context.Background(), loggables.Uuid("session"))
ctx := logging.ContextWithLoggable(context.Background(), newUUID("session"))
var err error

tp, err := tracing.NewTracerProvider(ctx)
Expand Down Expand Up @@ -109,7 +120,7 @@ func mainRet() (exitCode int) {
os.Args[1] = "version"
}

//Handle `ipfs help` and `ipfs help <sub-command>`
// Handle `ipfs help` and `ipfs help <sub-command>`
if os.Args[1] == "help" {
if len(os.Args) > 2 {
os.Args = append(os.Args[:1], os.Args[2:]...)
Expand Down
1 change: 1 addition & 0 deletions core/builder.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,7 @@ func NewNode(ctx context.Context, cfg *BuildCfg) (*IpfsNode, error) {
return nil, fmt.Errorf("building fx opts: %w", err)
}
}
//nolint:staticcheck // https://github.com/ipfs/kubo/pull/9423#issuecomment-1341038770
opts = append(opts, fx.Extract(n))

app := fx.New(opts...)
Expand Down
4 changes: 2 additions & 2 deletions core/node/libp2p/rcmgr.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ import (
"go.opencensus.io/stats/view"
"go.uber.org/fx"

config "github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/config"
"github.com/ipfs/kubo/core/node/helpers"
"github.com/ipfs/kubo/repo"
)
Expand Down Expand Up @@ -113,7 +113,7 @@ func ResourceManager(cfg config.SwarmConfig) interface{} {
manager = lrm
} else {
fmt.Println("go-libp2p resource manager protection disabled")
manager = network.NullResourceManager
manager = &network.NullResourceManager{}
}

opts.Opts = append(opts.Opts, libp2p.ResourceManager(manager))
Expand Down
8 changes: 2 additions & 6 deletions core/node/libp2p/smux.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,17 +19,13 @@ func yamuxTransport() network.Multiplexer {
if os.Getenv("YAMUX_DEBUG") != "" {
tpt.LogOutput = os.Stderr
}

return &tpt
}

func makeSmuxTransportOption(tptConfig config.Transports) (libp2p.Option, error) {
const yamuxID = "/yamux/1.0.0"
const mplexID = "/mplex/6.7.0"

ymxtpt := *yamux.DefaultTransport
ymxtpt.AcceptBacklog = 512

if prefs := os.Getenv("LIBP2P_MUX_PREFS"); prefs != "" {
// Using legacy LIBP2P_MUX_PREFS variable.
log.Error("LIBP2P_MUX_PREFS is now deprecated.")
Expand All @@ -47,7 +43,7 @@ func makeSmuxTransportOption(tptConfig config.Transports) (libp2p.Option, error)
}
switch tpt {
case yamuxID:
opts = append(opts, libp2p.Muxer(tpt, yamuxTransport))
opts = append(opts, libp2p.Muxer(tpt, yamuxTransport()))
case mplexID:
opts = append(opts, libp2p.Muxer(tpt, mplex.DefaultTransport))
default:
Expand All @@ -59,7 +55,7 @@ func makeSmuxTransportOption(tptConfig config.Transports) (libp2p.Option, error)
return prioritizeOptions([]priorityOption{{
priority: tptConfig.Multiplexers.Yamux,
defaultPriority: 100,
opt: libp2p.Muxer(yamuxID, yamuxTransport),
opt: libp2p.Muxer(yamuxID, yamuxTransport()),
}, {
priority: tptConfig.Multiplexers.Mplex,
defaultPriority: 200,
Expand Down
2 changes: 1 addition & 1 deletion core/node/libp2p/transport.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func Transports(tptConfig config.Transports) interface{} {
)
}
// TODO(9290): Make WithMetrics configurable
opts.Opts = append(opts.Opts, libp2p.Transport(quic.NewTransport, quic.WithMetrics()))
opts.Opts = append(opts.Opts, libp2p.Transport(quic.NewTransport))
}

// TODO(9292): Remove the false && to allows it enabled by default
Expand Down
14 changes: 8 additions & 6 deletions docs/changelogs/v0.18.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,8 @@ Below is an outline of all that is in this release, so you get a sense of all th
- [🔦 Highlights](#-highlights)
- [(DAG-)JSON and (DAG-)CBOR Response Formats on Gateways](#dag-json-and-dag-cbor-response-formats-on-gateways)
- [Content Routing](#content-routing)
- [Increased `Reprovider.Interval`](#increased-reproviderinterval)
- [Provider Record Republish and Expiration](#provider-record-republish-and-expiration)
- [Lowered `ConnMgr`](#lowered-connmgr)
- [Changelog](#changelog)
- [Contributors](#contributors)

Expand Down Expand Up @@ -88,13 +89,14 @@ Alternative routing rules, including alternative IPNI endpoints, can be configur

Learn more in [`Routing` docs](https://github.com/ipfs/kubo/blob/master/docs/config.md#routing).

#### Increased `Reprovider.Interval`
#### Provider Record Republish and Expiration

Default changed from 12h to 22h.
We also stopped `ipfs init` from hardcoding the default value in user config, allowing Kubo to adjust implicit default in future releases.
Default `Reprovider.Interval` changed from 12h to 22h to match new defaults for the Provider Record Expiration (48h) in [go-libp2p-kad-dht v0.20.0](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.20.0).

Rationale for increasing this can be found in [RFM 17: Provider Record Livenes Report](https://github.com/protocol/network-measurements/blob/master/results/rfm17-provider-record-liveness.md)
and [kubo#9326](https://github.com/ipfs/kubo/pull/9326).
Rationale for increasing this can be found in
[RFM 17: Provider Record Livenes Report](https://github.com/protocol/network-measurements/blob/master/results/rfm17-provider-record-liveness.md),
[kubo#9326](https://github.com/ipfs/kubo/pull/9326),
and the upstream DHT specifications at [libp2p/specs#451](https://github.com/libp2p/specs/pull/451).

Learn more: [`Reprovider` config](https://github.com/ipfs/go-ipfs/blob/master/docs/config.md#reprovider)

Expand Down
75 changes: 36 additions & 39 deletions docs/examples/kubo-as-a-library/go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,9 +8,9 @@ replace github.com/ipfs/kubo => ./../../..

require (
github.com/ipfs/go-ipfs-files v0.2.0
github.com/ipfs/interface-go-ipfs-core v0.7.0
github.com/ipfs/kubo v0.0.0-00010101000000-000000000000
github.com/libp2p/go-libp2p v0.23.4
github.com/ipfs/interface-go-ipfs-core v0.8.0
github.com/ipfs/kubo v0.14.0-rc1
github.com/libp2p/go-libp2p v0.24.0
github.com/multiformats/go-multiaddr v0.8.0
)

Expand All @@ -27,7 +27,7 @@ require (
github.com/cenkalti/backoff/v4 v4.1.3 // indirect
github.com/ceramicnetwork/go-dag-jose v0.1.0 // indirect
github.com/cespare/xxhash v1.1.0 // indirect
github.com/cespare/xxhash/v2 v2.1.2 // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/containerd/cgroups v1.0.4 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 // indirect
Expand All @@ -42,7 +42,6 @@ require (
github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 // indirect
github.com/flynn/noise v1.0.0 // indirect
github.com/francoispqt/gojay v1.2.13 // indirect
github.com/fsnotify/fsnotify v1.6.0 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect
Expand All @@ -52,6 +51,7 @@ require (
github.com/golang/protobuf v1.5.2 // indirect
github.com/golang/snappy v0.0.4 // indirect
github.com/google/gopacket v1.1.19 // indirect
github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/gorilla/mux v1.8.0 // indirect
github.com/gorilla/websocket v1.5.0 // indirect
Expand All @@ -63,9 +63,9 @@ require (
github.com/huin/goupnp v1.0.3 // indirect
github.com/ipfs/bbloom v0.0.4 // indirect
github.com/ipfs/go-bitfield v1.0.0 // indirect
github.com/ipfs/go-bitswap v0.10.2 // indirect
github.com/ipfs/go-bitswap v0.11.0 // indirect
github.com/ipfs/go-block-format v0.0.3 // indirect
github.com/ipfs/go-blockservice v0.4.0 // indirect
github.com/ipfs/go-blockservice v0.5.0 // indirect
github.com/ipfs/go-cid v0.3.2 // indirect
github.com/ipfs/go-cidutil v0.1.0 // indirect
github.com/ipfs/go-datastore v0.6.0 // indirect
Expand All @@ -77,34 +77,34 @@ require (
github.com/ipfs/go-fetcher v1.6.1 // indirect
github.com/ipfs/go-filestore v1.2.0 // indirect
github.com/ipfs/go-fs-lock v0.0.7 // indirect
github.com/ipfs/go-graphsync v0.13.1 // indirect
github.com/ipfs/go-graphsync v0.14.1-0.20221120210616-975d7aaeb15b // indirect
github.com/ipfs/go-ipfs-blockstore v1.2.0 // indirect
github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect
github.com/ipfs/go-ipfs-delay v0.0.1 // indirect
github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect
github.com/ipfs/go-ipfs-exchange-interface v0.2.0 // indirect
github.com/ipfs/go-ipfs-exchange-offline v0.3.0 // indirect
github.com/ipfs/go-ipfs-keystore v0.0.2 // indirect
github.com/ipfs/go-ipfs-keystore v0.1.0 // indirect
github.com/ipfs/go-ipfs-pinner v0.2.1 // indirect
github.com/ipfs/go-ipfs-posinfo v0.0.1 // indirect
github.com/ipfs/go-ipfs-pq v0.0.2 // indirect
github.com/ipfs/go-ipfs-provider v0.7.1 // indirect
github.com/ipfs/go-ipfs-routing v0.2.1 // indirect
github.com/ipfs/go-ipfs-provider v0.8.1 // indirect
github.com/ipfs/go-ipfs-routing v0.3.0 // indirect
github.com/ipfs/go-ipfs-util v0.0.2 // indirect
github.com/ipfs/go-ipld-cbor v0.0.5 // indirect
github.com/ipfs/go-ipld-format v0.4.0 // indirect
github.com/ipfs/go-ipld-git v0.1.1 // indirect
github.com/ipfs/go-ipld-legacy v0.1.1 // indirect
github.com/ipfs/go-ipns v0.3.0 // indirect
github.com/ipfs/go-libipfs v0.0.0-20221207180439-c7e7738575f9 // indirect
github.com/ipfs/go-libipfs v0.0.0-20221208220359-356ce09dd4a1 // indirect
github.com/ipfs/go-log v1.0.5 // indirect
github.com/ipfs/go-log/v2 v2.5.1 // indirect
github.com/ipfs/go-merkledag v0.8.1 // indirect
github.com/ipfs/go-metrics-interface v0.0.1 // indirect
github.com/ipfs/go-mfs v0.2.1 // indirect
github.com/ipfs/go-namesys v0.5.0 // indirect
github.com/ipfs/go-namesys v0.6.0 // indirect
github.com/ipfs/go-path v0.3.0 // indirect
github.com/ipfs/go-peertaskqueue v0.7.1 // indirect
github.com/ipfs/go-peertaskqueue v0.8.0 // indirect
github.com/ipfs/go-unixfs v0.4.1 // indirect
github.com/ipfs/go-unixfsnode v1.4.0 // indirect
github.com/ipfs/go-verifcid v0.0.2 // indirect
Expand All @@ -115,18 +115,17 @@ require (
github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect
github.com/jbenet/goprocess v0.1.4 // indirect
github.com/klauspost/compress v1.15.12 // indirect
github.com/klauspost/cpuid/v2 v2.1.2 // indirect
github.com/klauspost/cpuid/v2 v2.2.1 // indirect
github.com/koron/go-ssdp v0.0.3 // indirect
github.com/libp2p/go-buffer-pool v0.1.0 // indirect
github.com/libp2p/go-cidranger v1.1.0 // indirect
github.com/libp2p/go-doh-resolver v0.4.0 // indirect
github.com/libp2p/go-flow-metrics v0.1.0 // indirect
github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect
github.com/libp2p/go-libp2p-core v0.20.1 // indirect
github.com/libp2p/go-libp2p-kad-dht v0.18.0 // indirect
github.com/libp2p/go-libp2p-kbucket v0.4.7 // indirect
github.com/libp2p/go-libp2p-pubsub v0.8.1 // indirect
github.com/libp2p/go-libp2p-pubsub-router v0.5.0 // indirect
github.com/libp2p/go-libp2p-kad-dht v0.20.0 // indirect
github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect
github.com/libp2p/go-libp2p-pubsub v0.8.2 // indirect
github.com/libp2p/go-libp2p-pubsub-router v0.6.0 // indirect
github.com/libp2p/go-libp2p-record v0.2.0 // indirect
github.com/libp2p/go-libp2p-routing-helpers v0.4.1 // indirect
github.com/libp2p/go-libp2p-xor v0.1.0 // indirect
Expand All @@ -138,12 +137,12 @@ require (
github.com/libp2p/go-reuseport v0.2.0 // indirect
github.com/libp2p/go-yamux/v4 v4.0.0 // indirect
github.com/libp2p/zeroconf/v2 v2.2.0 // indirect
github.com/lucas-clemente/quic-go v0.29.1 // indirect
github.com/lucas-clemente/quic-go v0.31.0 // indirect
github.com/marten-seemann/qpack v0.3.0 // indirect
github.com/marten-seemann/qtls-go1-18 v0.1.3 // indirect
github.com/marten-seemann/qtls-go1-19 v0.1.1 // indirect
github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect
github.com/marten-seemann/webtransport-go v0.1.1 // indirect
github.com/marten-seemann/webtransport-go v0.4.1 // indirect
github.com/mattn/go-isatty v0.0.16 // indirect
github.com/mattn/go-pointer v0.0.1 // indirect
github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect
Expand All @@ -154,16 +153,15 @@ require (
github.com/mitchellh/go-homedir v1.1.0 // indirect
github.com/mr-tron/base58 v1.2.0 // indirect
github.com/multiformats/go-base32 v0.1.0 // indirect
github.com/multiformats/go-base36 v0.1.0 // indirect
github.com/multiformats/go-base36 v0.2.0 // indirect
github.com/multiformats/go-multiaddr-dns v0.3.1 // indirect
github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect
github.com/multiformats/go-multibase v0.1.1 // indirect
github.com/multiformats/go-multicodec v0.7.0 // indirect
github.com/multiformats/go-multihash v0.2.1 // indirect
github.com/multiformats/go-multistream v0.3.3 // indirect
github.com/multiformats/go-varint v0.0.6 // indirect
github.com/nxadm/tail v1.4.8 // indirect
github.com/onsi/ginkgo v1.16.5 // indirect
github.com/multiformats/go-varint v0.0.7 // indirect
github.com/onsi/ginkgo/v2 v2.5.1 // indirect
github.com/opencontainers/runtime-spec v1.0.2 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
github.com/openzipkin/zipkin-go v0.4.0 // indirect
Expand All @@ -178,14 +176,14 @@ require (
github.com/samber/lo v1.36.0 // indirect
github.com/spacemonkeygo/spacelog v0.0.0-20180420211403-2296661a0572 // indirect
github.com/spaolacci/murmur3 v1.1.0 // indirect
github.com/syndtr/goleveldb v1.0.0 // indirect
github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect
github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc // indirect
github.com/whyrusleeping/cbor-gen v0.0.0-20210219115102-f37d292932f2 // indirect
github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f // indirect
github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect
github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7 // indirect
github.com/whyrusleeping/timecache v0.0.0-20160911033111-cfcb2f1abfee // indirect
go.opencensus.io v0.23.0 // indirect
go.opencensus.io v0.24.0 // indirect
go.opentelemetry.io/otel v1.7.0 // indirect
go.opentelemetry.io/otel/exporters/jaeger v1.7.0 // indirect
go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.7.0 // indirect
Expand All @@ -198,24 +196,23 @@ require (
go.opentelemetry.io/otel/trace v1.7.0 // indirect
go.opentelemetry.io/proto/otlp v0.16.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/dig v1.14.1 // indirect
go.uber.org/fx v1.17.1 // indirect
go.uber.org/dig v1.15.0 // indirect
go.uber.org/fx v1.18.2 // indirect
go.uber.org/multierr v1.8.0 // indirect
go.uber.org/zap v1.23.0 // indirect
go.uber.org/zap v1.24.0 // indirect
go4.org v0.0.0-20200411211856-f5505b9728dd // indirect
golang.org/x/crypto v0.1.0 // indirect
golang.org/x/exp v0.0.0-20221106115401-f9659909a136 // indirect
golang.org/x/mod v0.6.0 // indirect
golang.org/x/net v0.1.0 // indirect
golang.org/x/crypto v0.3.0 // indirect
golang.org/x/exp v0.0.0-20221205204356-47842c84f3db // indirect
golang.org/x/mod v0.7.0 // indirect
golang.org/x/net v0.3.0 // indirect
golang.org/x/sync v0.1.0 // indirect
golang.org/x/sys v0.2.0 // indirect
golang.org/x/text v0.4.0 // indirect
golang.org/x/tools v0.2.0 // indirect
golang.org/x/sys v0.3.0 // indirect
golang.org/x/text v0.5.0 // indirect
golang.org/x/tools v0.3.0 // indirect
golang.org/x/xerrors v0.0.0-20220609144429-65e65417b02f // indirect
google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 // indirect
google.golang.org/grpc v1.47.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/square/go-jose.v2 v2.5.1 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
lukechampine.com/blake3 v1.1.7 // indirect
)
Loading