lightning may stuck at gRPC RecvMsg

[lance6716]

Bug Report

We found this goroutine stack

11 @ 0x1c2d6ee 0x1c3e185 0x21cf23c 0x225699a 0x2256987 0x2255c1f 0x22548da 0x2255a73 0x3d9bb96 0x3d999f1 0x3d98066 0x3d8942b 0x3d88f47 0x3d8be0e 0x3ac2a96 0x1c625c1
#	0x21cf23b	google.golang.org/grpc/internal/transport.(*Stream).waitOnHeader+0x7b			/go/pkg/mod/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:327
#	0x2256999	google.golang.org/grpc/internal/transport.(*Stream).RecvCompress+0xb9			/go/pkg/mod/google.golang.org/grpc@v1.59.0/internal/transport/transport.go:342
#	0x2256986	google.golang.org/grpc.(*csAttempt).recvMsg+0xa6					/go/pkg/mod/google.golang.org/grpc@v1.59.0/stream.go:1070
#	0x2255c1e	google.golang.org/grpc.(*clientStream).RecvMsg.func1+0x1e				/go/pkg/mod/google.golang.org/grpc@v1.59.0/stream.go:927
#	0x22548d9	google.golang.org/grpc.(*clientStream).withRetry+0x139					/go/pkg/mod/google.golang.org/grpc@v1.59.0/stream.go:776
#	0x2255a72	google.golang.org/grpc.(*clientStream).RecvMsg+0x112					/go/pkg/mod/google.golang.org/grpc@v1.59.0/stream.go:926
#	0x3d9bb95	github.com/pingcap/tidb/br/pkg/lightning/backend/local.(*Backend).doWrite.func5+0x2b5	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/br/pkg/lightning/backend/local/region_job.go:354
#	0x3d999f0	github.com/pingcap/tidb/br/pkg/lightning/backend/local.(*Backend).doWrite+0x1890	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/br/pkg/lightning/backend/local/region_job.go:389
#	0x3d98065	github.com/pingcap/tidb/br/pkg/lightning/backend/local.(*Backend).writeToTiKV+0x25	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/br/pkg/lightning/backend/local/region_job.go:189
#	0x3d8942a	github.com/pingcap/tidb/br/pkg/lightning/backend/local.(*Backend).executeJob+0xaa	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/br/pkg/lightning/backend/local/local.go:1435
#	0x3d88f46	github.com/pingcap/tidb/br/pkg/lightning/backend/local.(*Backend).startWorker+0x1c6	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/br/pkg/lightning/backend/local/local.go:1344
#	0x3d8be0d	github.com/pingcap/tidb/br/pkg/lightning/backend/local.(*Backend).doImport.func5+0x2d	/home/jenkins/agent/workspace/build-common/go/src/github.com/pingcap/tidb/br/pkg/lightning/backend/local/local.go:1677
#	0x3ac2a95	golang.org/x/sync/errgroup.(*Group).Go.func1+0x55					/go/pkg/mod/golang.org/x/sync@v0.4.0/errgroup/errgroup.go:75