Update dependencies. Requires go 1.18 and BasicLimiter -> ScalingLimitConfig

.github/workflows/build.yml

@@ -85,11 +85,11 @@ jobs:
             OS: darwin
             ARCH: amd64
     steps:
-    - name: Set up Go 1.17
+    - name: Set up Go 1.18
       uses: actions/setup-go@v1
       id: go
       with:
-        go-version: 1.17
+        go-version: 1.18
     - uses: actions/checkout@v1
       with:
         submodules: true

.github/workflows/go.yml

@@ -16,7 +16,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: 1.17
+        go-version: 1.18
 
     - name: Set up Rust
       uses: actions-rs/toolchain@v1

autoretrieve/autoretrieve.go

@@ -17,9 +17,9 @@ import (
 	"github.com/ipfs/go-cid"
 	"github.com/ipfs/go-datastore"
 	logging "github.com/ipfs/go-log/v2"
-	"github.com/libp2p/go-libp2p-core/crypto"
-	"github.com/libp2p/go-libp2p-core/host"
-	"github.com/libp2p/go-libp2p-core/peer"
+	"github.com/libp2p/go-libp2p/core/crypto"
+	"github.com/libp2p/go-libp2p/core/host"
+	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/multiformats/go-multiaddr"
 	"github.com/multiformats/go-multihash"
 	"gorm.io/gorm"

autoretrieve/engine.go

@@ -26,7 +26,7 @@ import (
 	logging "github.com/ipfs/go-log/v2"
 	"github.com/ipld/go-ipld-prime"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
-	"github.com/libp2p/go-libp2p-core/peer"
+	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/multiformats/go-multiaddr"
 	"gorm.io/gorm"
 )

autoretrieve/options.go

@@ -8,10 +8,10 @@ import (
 	"github.com/ipfs/go-datastore"
 	dssync "github.com/ipfs/go-datastore/sync"
 	"github.com/libp2p/go-libp2p"
-	"github.com/libp2p/go-libp2p-core/crypto"
-	"github.com/libp2p/go-libp2p-core/host"
-	"github.com/libp2p/go-libp2p-core/peer"
 	pubsub "github.com/libp2p/go-libp2p-pubsub"
+	"github.com/libp2p/go-libp2p/core/crypto"
+	"github.com/libp2p/go-libp2p/core/host"
+	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/multiformats/go-multiaddr"
 )
 

bwgrapher.go

@@ -4,7 +4,7 @@ import (
 	"time"
 
 	influxdb2 "github.com/influxdata/influxdb-client-go/v2"
-	"github.com/libp2p/go-libp2p-core/metrics"
+	"github.com/libp2p/go-libp2p/core/metrics"
 )
 
 func StartBandwidthGrapher(client influxdb2.Client, bwc *metrics.BandwidthCounter) {

cmd/bsget/main.go

@@ -14,8 +14,8 @@ import (
 	ipld "github.com/ipfs/go-ipld-format"
 	"github.com/ipfs/go-merkledag"
 	"github.com/libp2p/go-libp2p"
-	"github.com/libp2p/go-libp2p-core/peer"
 	rhelp "github.com/libp2p/go-libp2p-routing-helpers"
+	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/multiformats/go-multiaddr"
 
 	"github.com/cheggaaa/pb/v3"

cmd/estuary-shuttle/main.go

@@ -34,7 +34,7 @@ import (
 	semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
 	"go.opentelemetry.io/otel/trace"
 
-	"github.com/libp2p/go-libp2p-core/peer"
+	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/urfave/cli/v2"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
@@ -70,7 +70,7 @@ import (
 	"github.com/ipld/go-car"
 	"github.com/labstack/echo/v4"
 	"github.com/labstack/echo/v4/middleware"
-	rcmgr "github.com/libp2p/go-libp2p-resource-manager"
+	rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
 	routed "github.com/libp2p/go-libp2p/p2p/host/routed"
 	"github.com/whyrusleeping/memo"
 )
@@ -101,6 +101,7 @@ func before(cctx *cli.Context) error {
 	_ = logging.SetLogLevel("bs-wal", level)
 	_ = logging.SetLogLevel("bs-migrate", level)
 	_ = logging.SetLogLevel("rcmgr", level)
+	_ = logging.SetLogLevel("est-node", level)
 
 	return nil
 }

cmd/estuary-shuttle/rpc.go

@@ -18,7 +18,7 @@ import (
 	"github.com/ipfs/go-cid"
 	cbor "github.com/ipfs/go-ipld-cbor"
 	"github.com/ipfs/go-merkledag"
-	"github.com/libp2p/go-libp2p-core/peer"
+	"github.com/libp2p/go-libp2p/core/peer"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/xerrors"

config/config_test.go

@@ -1,12 +1,13 @@
 package config
 
 import (
+	"github.com/libp2p/go-libp2p"
 	"os"
 	"path/filepath"
 	"testing"
 
-	"github.com/libp2p/go-libp2p-core/network"
-	rcmgr "github.com/libp2p/go-libp2p-resource-manager"
+	"github.com/libp2p/go-libp2p/core/network"
+	rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -17,23 +18,22 @@ func checkNodeConfig(t *testing.T, node *Node) {
 	assert.NotEmpty(node.DatastoreDir)
 	assert.NotEmpty(node.WalletDir)
 
-	assert.Greater(node.Limits.SystemLimit.Conns, 0)
-	assert.Greater(node.Limits.SystemLimit.ConnsInbound, 0)
-	assert.Greater(node.Limits.SystemLimit.ConnsOutbound, 0)
-	assert.Greater(node.Limits.SystemLimit.Streams, 0)
-	assert.Greater(node.Limits.SystemLimit.StreamsInbound, 0)
-	assert.Greater(node.Limits.SystemLimit.StreamsOutbound, 0)
-	assert.Greater(node.Limits.SystemLimit.FD, 0)
-	assert.Greater(node.Limits.SystemLimit.MinMemory, int64(0))
-	assert.Greater(node.Limits.SystemLimit.MaxMemory, node.Limits.SystemLimit.MinMemory)
-
-	assert.Greater(node.Limits.TransientLimit.Conns, 0)
-	assert.Greater(node.Limits.TransientLimit.ConnsInbound, 0)
-	assert.Greater(node.Limits.TransientLimit.ConnsOutbound, 0)
-	assert.Greater(node.Limits.TransientLimit.Streams, 0)
-	assert.Greater(node.Limits.TransientLimit.StreamsInbound, 0)
-	assert.Greater(node.Limits.TransientLimit.StreamsOutbound, 0)
-	assert.Greater(node.Limits.TransientLimit.FD, 0)
+	assert.Greater(node.Limits.SystemBaseLimit.Conns, 0)
+	assert.Greater(node.Limits.SystemBaseLimit.ConnsInbound, 0)
+	assert.Greater(node.Limits.SystemBaseLimit.ConnsOutbound, 0)
+	assert.Greater(node.Limits.SystemBaseLimit.Streams, 0)
+	assert.Greater(node.Limits.SystemBaseLimit.StreamsInbound, 0)
+	assert.Greater(node.Limits.SystemBaseLimit.StreamsOutbound, 0)
+	assert.Greater(node.Limits.SystemBaseLimit.FD, 0)
+	assert.Greater(node.Limits.SystemBaseLimit.Memory, int64(0))
+
+	assert.Greater(node.Limits.TransientBaseLimit.Conns, 0)
+	assert.Greater(node.Limits.TransientBaseLimit.ConnsInbound, 0)
+	assert.Greater(node.Limits.TransientBaseLimit.ConnsOutbound, 0)
+	assert.Greater(node.Limits.TransientBaseLimit.Streams, 0)
+	assert.Greater(node.Limits.TransientBaseLimit.StreamsInbound, 0)
+	assert.Greater(node.Limits.TransientBaseLimit.StreamsOutbound, 0)
+	assert.Greater(node.Limits.TransientBaseLimit.FD, 0)
 
 	assert.NotEmpty(node.ListenAddrs)
 	assert.NotEmpty(node.ListenAddrs[0])
@@ -76,23 +76,28 @@ func TestApplyLimits(t *testing.T) {
 	assert := assert.New(t)
 	config := NewShuttle("test-version").Node.Limits
 
-	limiter := rcmgr.NewDefaultLimiter()
-	config.apply(limiter)
-
-	assert.Equal(limiter.SystemLimits.GetConnLimit(network.DirInbound), config.SystemLimit.ConnsInbound)
-	assert.Equal(limiter.SystemLimits.GetConnLimit(network.DirOutbound), config.SystemLimit.ConnsOutbound)
-	assert.Equal(limiter.SystemLimits.GetConnTotalLimit(), config.SystemLimit.Conns)
-	assert.Equal(limiter.TransientLimits.GetConnLimit(network.DirInbound), config.TransientLimit.ConnsInbound)
-	assert.Equal(limiter.TransientLimits.GetConnLimit(network.DirOutbound), config.TransientLimit.ConnsOutbound)
-	assert.Equal(limiter.TransientLimits.GetConnTotalLimit(), config.TransientLimit.Conns)
-	assert.Equal(limiter.SystemLimits.GetStreamLimit(network.DirInbound), config.SystemLimit.StreamsInbound)
-	assert.Equal(limiter.SystemLimits.GetStreamLimit(network.DirOutbound), config.SystemLimit.StreamsOutbound)
-	assert.Equal(limiter.SystemLimits.GetStreamTotalLimit(), config.SystemLimit.Streams)
-	assert.Equal(limiter.TransientLimits.GetStreamLimit(network.DirInbound), config.TransientLimit.StreamsInbound)
-	assert.Equal(limiter.TransientLimits.GetStreamLimit(network.DirOutbound), config.TransientLimit.StreamsOutbound)
-	assert.Equal(limiter.TransientLimits.GetStreamTotalLimit(), config.TransientLimit.Streams)
-	assert.Equal(limiter.SystemLimits.GetFDLimit(), config.SystemLimit.FD)
-	assert.Equal(limiter.TransientLimits.GetFDLimit(), config.TransientLimit.FD)
+	libp2p.SetDefaultServiceLimits(&config)
+	limitConfig := config.AutoScale()
+	limitConfig.Apply(rcmgr.DefaultLimits.AutoScale())
+	limiter := rcmgr.NewFixedLimiter(limitConfig)
+
+	systemLimits := limiter.GetSystemLimits()
+	transientLimits := limiter.GetTransientLimits()
+
+	assert.Equal(systemLimits.GetConnLimit(network.DirInbound), config.SystemBaseLimit.ConnsInbound)
+	assert.Equal(systemLimits.GetConnLimit(network.DirOutbound), config.SystemBaseLimit.ConnsOutbound)
+	assert.Equal(systemLimits.GetConnTotalLimit(), config.SystemBaseLimit.Conns)
+	assert.Equal(transientLimits.GetConnLimit(network.DirInbound), config.TransientBaseLimit.ConnsInbound)
+	assert.Equal(transientLimits.GetConnLimit(network.DirOutbound), config.TransientBaseLimit.ConnsOutbound)
+	assert.Equal(transientLimits.GetConnTotalLimit(), config.TransientBaseLimit.Conns)
+	assert.Equal(systemLimits.GetStreamLimit(network.DirInbound), config.SystemBaseLimit.StreamsInbound)
+	assert.Equal(systemLimits.GetStreamLimit(network.DirOutbound), config.SystemBaseLimit.StreamsOutbound)
+	assert.Equal(systemLimits.GetStreamTotalLimit(), config.SystemBaseLimit.Streams)
+	assert.Equal(transientLimits.GetStreamLimit(network.DirInbound), config.TransientBaseLimit.StreamsInbound)
+	assert.Equal(transientLimits.GetStreamLimit(network.DirOutbound), config.TransientBaseLimit.StreamsOutbound)
+	assert.Equal(transientLimits.GetStreamTotalLimit(), config.TransientBaseLimit.Streams)
+	assert.Equal(systemLimits.GetFDLimit(), config.SystemBaseLimit.FD)
+	assert.Equal(transientLimits.GetFDLimit(), config.TransientBaseLimit.FD)
 }
 
 func TestEstuaryJSONRoundtrip(t *testing.T) {

config/deal.go

@@ -3,7 +3,7 @@ package config
 import (
 	"github.com/application-research/filclient"
 	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/libp2p/go-libp2p-core/protocol"
+	"github.com/libp2p/go-libp2p/core/protocol"
 )
 
 const (

config/estuary.go

@@ -1,14 +1,15 @@
 package config
 
 import (
+	rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager"
 	"path/filepath"
 	"time"
 
 	"github.com/application-research/estuary/build"
 	"github.com/application-research/estuary/node/modules/peering"
 	"github.com/application-research/filclient"
 	"github.com/filecoin-project/go-state-types/abi"
-	"github.com/libp2p/go-libp2p-core/protocol"
+	"github.com/libp2p/go-libp2p/core/protocol"
 )
 
 type Estuary struct {
@@ -133,32 +134,39 @@ func NewEstuary(appVersion string) *Estuary {
 			},
 
 			NoLimiter: true,
-			Limits: Limits{
-				SystemLimit: SystemLimit{
-					MinMemory:      1 << 30,
-					MaxMemory:      10 << 30,
-					MemoryFraction: .2,
-
+			Limits: rcmgr.ScalingLimitConfig{
+				SystemBaseLimit: rcmgr.BaseLimit{
+					Memory:          10 << 30,
 					StreamsInbound:  64 << 10,
 					StreamsOutbound: 128 << 10,
 					Streams:         256 << 10,
-
-					ConnsInbound:  256,
-					ConnsOutbound: 256,
-					Conns:         1024,
-
-					FD: 8192,
+					ConnsInbound:    256,
+					ConnsOutbound:   256,
+					Conns:           1024,
+					FD:              8192,
 				},
-				TransientLimit: TransientLimit{
+				TransientBaseLimit: rcmgr.BaseLimit{
+					Memory:          4096,
 					StreamsInbound:  2 << 10,
 					StreamsOutbound: 4 << 10,
 					Streams:         4 << 10,
-
-					ConnsInbound:  256,
-					ConnsOutbound: 256,
-					Conns:         512,
-
-					FD: 1024,
+					ConnsInbound:    256,
+					ConnsOutbound:   256,
+					Conns:           512,
+					FD:              1024,
+				},
+				// TODO: remove after https://github.com/libp2p/go-libp2p/pull/1878 is released
+				ServicePeerBaseLimit: rcmgr.BaseLimit{
+					StreamsInbound:  128,
+					StreamsOutbound: 256,
+					Streams:         256,
+					Memory:          16 << 20,
+				},
+				ServicePeerLimitIncrease: rcmgr.BaseLimitIncrease{
+					StreamsInbound:  4,
+					StreamsOutbound: 8,
+					Streams:         8,
+					Memory:          4 << 20,
 				},
 			},
 			ConnectionManager: ConnectionManager{