Merge pull request #63977 from runcom/increase-grpc-resp-size

Automatic merge from submit-queue (batch tested with PRs 60012, 63692, 63977, 63960, 64008). If you want to cherry-pick this change to another branch, please follow the instructions <a href="https://github.com/kubernetes/community/blob/master/contributors/devel/cherry-picks.md">here</a>.

pkg: kubelet: remote: increase grpc client default size

Signed-off-by: Antonio Murdaca <runcom@redhat.com>



**What this PR does / why we need it**:

when running lots and lots of containers and having tons of images on a given node, we started seeing this in the logs (with docker):

```
Unable to retrieve pods: rpc error: code = ResourceExhausted desc = grpc: received message larger than max (4208374 vs. 4194304)
```

That's because the grpc client is defaulting to a 4MB response size.
This patch increases the resp size to 8MB to avoid such issue.

**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
Fixes #

**Special notes for your reviewer**:

**Release note**:

```release-note
increase grpc client default response size
```

pkg/kubelet/remote/remote_image.go

@@ -43,7 +43,7 @@ func NewRemoteImageService(endpoint string, connectionTimeout time.Duration) (in
 		return nil, err
 	}
 
-	conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithTimeout(connectionTimeout), grpc.WithDialer(dailer))
+	conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithTimeout(connectionTimeout), grpc.WithDialer(dailer), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
 	if err != nil {
 		glog.Errorf("Connect remote image service %s failed: %v", addr, err)
 		return nil, err

pkg/kubelet/remote/remote_runtime.go

@@ -45,7 +45,7 @@ func NewRemoteRuntimeService(endpoint string, connectionTimeout time.Duration) (
 	if err != nil {
 		return nil, err
 	}
-	conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithTimeout(connectionTimeout), grpc.WithDialer(dailer))
+	conn, err := grpc.Dial(addr, grpc.WithInsecure(), grpc.WithTimeout(connectionTimeout), grpc.WithDialer(dailer), grpc.WithDefaultCallOptions(grpc.MaxCallRecvMsgSize(maxMsgSize)))
 	if err != nil {
 		glog.Errorf("Connect remote runtime %s failed: %v", addr, err)
 		return nil, err

pkg/kubelet/remote/utils.go

@@ -24,6 +24,10 @@ import (
 	runtimeapi "k8s.io/kubernetes/pkg/kubelet/apis/cri/runtime/v1alpha2"
 )
 
+// maxMsgSize use 8MB as the default message size limit.
+// grpc library default is 4MB
+const maxMsgSize = 1024 * 1024 * 8
+
 // getContextWithTimeout returns a context with timeout.
 func getContextWithTimeout(timeout time.Duration) (context.Context, context.CancelFunc) {
 	return context.WithTimeout(context.Background(), timeout)