Sdk memory optimizations (#2622)

CHANGELOG.md

@@ -20,6 +20,7 @@ Changelog for NeoFS Node
 - BoltDB open timeout increased from 100ms to 1s (#2499)
 - Internal container cache size from 10 to 1000 (#2600)
 - Transaction witness scope no longer uses CustomGroups relying on more strict Rules now (#2619)
+- New optimized SDK version is integrated (#2622)
 
 ### Removed
 - deprecated `no-precheck` flag of `neofs-cli container set-eacl` (#2496)

cmd/neofs-node/config.go

@@ -536,11 +536,18 @@ func initCfg(appCfg *config.Config) *cfg {
 	)
 	fatalOnErr(err)
 
+	var buffers sync.Pool
+	buffers.New = func() any {
+		b := make([]byte, cache.DefaultBufferSize)
+		return &b
+	}
+
 	cacheOpts := cache.ClientCacheOpts{
 		DialTimeout:      apiclientconfig.DialTimeout(appCfg),
 		StreamTimeout:    apiclientconfig.StreamTimeout(appCfg),
 		AllowExternal:    apiclientconfig.AllowExternal(appCfg),
 		ReconnectTimeout: apiclientconfig.ReconnectTimeout(appCfg),
+		Buffers:          &buffers,
 	}
 	c.shared = shared{
 		key:            key,

go.mod

@@ -18,7 +18,7 @@ require (
 	github.com/nspcc-dev/neo-go v0.102.0
 	github.com/nspcc-dev/neofs-api-go/v2 v2.14.0
 	github.com/nspcc-dev/neofs-contract v0.18.0
-	github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20230926161529-a5cb78a74aed
+	github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20231011074223-dd4e3e09a100
 	github.com/nspcc-dev/tzhash v1.7.1
 	github.com/olekukonko/tablewriter v0.0.5
 	github.com/panjf2000/ants/v2 v2.8.2
@@ -52,7 +52,7 @@ require (
 	github.com/golang/snappy v0.0.3 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
-	github.com/hashicorp/golang-lru v0.6.0 // indirect
+	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/holiman/uint256 v1.2.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect

go.sum

@@ -167,8 +167,8 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
-github.com/hashicorp/golang-lru v0.6.0 h1:uL2shRDx7RTrOrTCUZEGP/wJUFiUI8QT6E7z5o8jga4=
-github.com/hashicorp/golang-lru v0.6.0/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
+github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c=
+github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
 github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
@@ -258,8 +258,8 @@ github.com/nspcc-dev/neofs-contract v0.18.0 h1:9g50b16s0mQFFskG93yRSWh4KL7yYOW+x
 github.com/nspcc-dev/neofs-contract v0.18.0/go.mod h1:UQr1rUjg0eibLwJd6vfsJJEUBnmRysCg8XQd1HYiS2w=
 github.com/nspcc-dev/neofs-crypto v0.4.0 h1:5LlrUAM5O0k1+sH/sktBtrgfWtq1pgpDs09fZo+KYi4=
 github.com/nspcc-dev/neofs-crypto v0.4.0/go.mod h1:6XJ8kbXgOfevbI2WMruOtI+qUJXNwSGM/E9eClXxPHs=
-github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20230926161529-a5cb78a74aed h1:ySOlpzLNU3djblNtjZFiTrfoE9zZ5fd1bwjxbbI3gvM=
-github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20230926161529-a5cb78a74aed/go.mod h1:W+ImTNRnSNMH8w43H1knCcIqwu7dLHePXtlJNZ7EFIs=
+github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20231011074223-dd4e3e09a100 h1:1m/7YyGJk32IEAl4/NFKY6Li/iqZlQyhpLR5MhUtQts=
+github.com/nspcc-dev/neofs-sdk-go v1.0.0-rc.11.0.20231011074223-dd4e3e09a100/go.mod h1:4LToeC6jfed7PWoJXDTaj9sS4W+C1xm2SbCe709VR8U=
 github.com/nspcc-dev/rfc6979 v0.2.0 h1:3e1WNxrN60/6N0DW7+UYisLeZJyfqZTNOjeV/toYvOE=
 github.com/nspcc-dev/rfc6979 v0.2.0/go.mod h1:exhIh1PdpDC5vQmyEsGvc4YDM/lyQp/452QxGq/UEso=
 github.com/nspcc-dev/tzhash v1.7.1 h1:6zmexLqdTF/ssbUAh7XJS7RxgKWaw28kdNpE/4UFdEU=

pkg/innerring/innerring.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"io"
 	"net"
+	"sync"
 	"sync/atomic"
 
 	"github.com/nspcc-dev/neo-go/pkg/core/block"
@@ -42,6 +43,7 @@ import (
 	repClient "github.com/nspcc-dev/neofs-node/pkg/morph/client/reputation"
 	"github.com/nspcc-dev/neofs-node/pkg/morph/event"
 	"github.com/nspcc-dev/neofs-node/pkg/morph/timer"
+	"github.com/nspcc-dev/neofs-node/pkg/network/cache"
 	audittask "github.com/nspcc-dev/neofs-node/pkg/services/audit/taskmanager"
 	control "github.com/nspcc-dev/neofs-node/pkg/services/control/ir"
 	controlsrv "github.com/nspcc-dev/neofs-node/pkg/services/control/ir/server"
@@ -588,13 +590,20 @@ func New(ctx context.Context, log *zap.Logger, cfg *viper.Viper, errChan chan<-
 		cfg.GetDuration("indexer.cache_timeout"),
 	)
 
+	var buffers sync.Pool
+	buffers.New = func() any {
+		b := make([]byte, cache.DefaultBufferSize)
+		return &b
+	}
+
 	clientCache := newClientCache(&clientCacheParams{
 		Log:           log,
 		Key:           &server.key.PrivateKey,
 		SGTimeout:     cfg.GetDuration("audit.timeout.get"),
 		HeadTimeout:   cfg.GetDuration("audit.timeout.head"),
 		RangeTimeout:  cfg.GetDuration("audit.timeout.rangehash"),
 		AllowExternal: cfg.GetBool("audit.allow_external"),
+		Buffers:       &buffers,
 	})
 
 	server.registerNoErrCloser(clientCache.cache.CloseAll)

pkg/innerring/rpc.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"crypto/ecdsa"
 	"fmt"
+	"sync"
 	"time"
 
 	clientcore "github.com/nspcc-dev/neofs-node/pkg/core/client"
@@ -40,13 +41,15 @@ type (
 		AllowExternal bool
 
 		SGTimeout, HeadTimeout, RangeTimeout time.Duration
+
+		Buffers *sync.Pool
 	}
 )
 
 func newClientCache(p *clientCacheParams) *ClientCache {
 	return &ClientCache{
 		log:          p.Log,
-		cache:        cache.NewSDKClientCache(cache.ClientCacheOpts{AllowExternal: p.AllowExternal}),
+		cache:        cache.NewSDKClientCache(cache.ClientCacheOpts{AllowExternal: p.AllowExternal, Buffers: p.Buffers}),
 		key:          p.Key,
 		sgTimeout:    p.SGTimeout,
 		headTimeout:  p.HeadTimeout,

pkg/network/cache/client.go

@@ -8,6 +8,9 @@ import (
 	"github.com/nspcc-dev/neofs-sdk-go/client"
 )
 
+// DefaultBufferSize describes default max GRPC message size. Unfortunately GRPC lib contains this const in private.
+const DefaultBufferSize = 4 * 1024 * 1024 // 4MB
+
 type (
 	// ClientCache is a structure around neofs-sdk-go/client to reuse
 	// already created clients.
@@ -23,6 +26,7 @@ type (
 		ReconnectTimeout time.Duration
 		ResponseCallback func(client.ResponseMetaInfo) error
 		AllowExternal    bool
+		Buffers          *sync.Pool
 	}
 )
 

pkg/network/cache/multi.go

@@ -58,6 +58,8 @@ func (x *multiClient) createForAddress(addr network.Address) (clientcore.Client,
 		prmDial client.PrmDial
 	)
 
+	prmInit.SetSignMessageBuffers(x.opts.Buffers)
+
 	prmDial.SetServerURI(addr.URIAddr())
 
 	if x.opts.DialTimeout > 0 {