Flink connector throw exception when reading region

[gbzy]

Flink connector throw exception when reading table. The number of records is about 50 million.

TIDB VERSION
v4.0.9

POM CONFIG

<dependency>
        <groupId>io.tidb</groupId>
        <artifactId>flink-tidb-connector-1.13</artifactId>
        <version>0.0.4</version>
</dependency>

DDL

create table if not exists test (
    `id` string
  )
with (
    'connector' = 'tidb',
    'tidb.database.url' = 'jdbc:mysql://ip:port/database',
    'tidb.table.name' = 'test',
    'tidb.database.name' = 'test',
    'tidb.username' = 'test',
    'tidb.password' = 'test'

EXCEPTION

org.tikv.common.exception.TiClientInternalException: Error reading region:
	at org.tikv.common.operation.iterator.DAGIterator.doReadNextRegionChunks(DAGIterator.java:184)
	at org.tikv.common.operation.iterator.DAGIterator.readNextRegionChunks(DAGIterator.java:161)
	at org.tikv.common.operation.iterator.DAGIterator.hasNext(DAGIterator.java:107)
	at io.tidb.bigdata.tidb.RecordCursorInternal.advanceNextPosition(RecordCursorInternal.java:49)
	at io.tidb.bigdata.flink.connector.source.reader.TiDBSourceSplitRecords.nextRecordFromSplit(TiDBSourceSplitRecords.java:74)
	at io.tidb.bigdata.flink.connector.source.reader.TiDBSourceSplitRecords.nextRecordFromSplit(TiDBSourceSplitRecords.java:35)
	at org.apache.flink.connector.base.source.reader.SourceReaderBase.pollNext(SourceReaderBase.java:125)
	at org.apache.flink.streaming.api.operators.SourceOperator.emitNext(SourceOperator.java:294)
	at org.apache.flink.streaming.runtime.io.StreamTaskSourceInput.emitNext(StreamTaskSourceInput.java:69)
	at org.apache.flink.streaming.runtime.io.StreamOneInputProcessor.processInput(StreamOneInputProcessor.java:66)
	at org.apache.flink.streaming.runtime.tasks.StreamTask.processInput(StreamTask.java:423)
	at org.apache.flink.streaming.runtime.tasks.mailbox.MailboxProcessor.runMailboxLoop(MailboxProcessor.java:204)
	at org.apache.flink.streaming.runtime.tasks.StreamTask.runMailboxLoop(StreamTask.java:681)
	at org.apache.flink.streaming.runtime.tasks.StreamTask.executeInvoke(StreamTask.java:636)
	at org.apache.flink.streaming.runtime.tasks.StreamTask.runWithCleanUpOnFail(StreamTask.java:647)
	at org.apache.flink.streaming.runtime.tasks.StreamTask.invoke(StreamTask.java:620)
	at org.apache.flink.runtime.taskmanager.Task.doRun(Task.java:779)
	at org.apache.flink.runtime.taskmanager.Task.run(Task.java:566)
	at java.lang.Thread.run(Thread.java:748)
Caused by: java.util.concurrent.ExecutionException: org.tikv.common.exception.RegionTaskException: Handle region task failed:
	at java.util.concurrent.FutureTask.report(FutureTask.java:122)
	at java.util.concurrent.FutureTask.get(FutureTask.java:192)
	at org.tikv.common.operation.iterator.DAGIterator.doReadNextRegionChunks(DAGIterator.java:179)
	... 18 more
Caused by: org.tikv.common.exception.RegionTaskException: Handle region task failed:
	at org.tikv.common.operation.iterator.DAGIterator.process(DAGIterator.java:227)
	at org.tikv.common.operation.iterator.DAGIterator.lambda$submitTasks$1(DAGIterator.java:85)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	... 1 more
Caused by: org.tikv.common.exception.GrpcException: retry is exhausted.
	at org.tikv.common.util.ConcreteBackOffer.doBackOffWithMaxSleep(ConcreteBackOffer.java:148)
	at org.tikv.common.util.ConcreteBackOffer.doBackOff(ConcreteBackOffer.java:119)
	at org.tikv.common.operation.RegionErrorHandler.handleRequestError(RegionErrorHandler.java:176)
	at org.tikv.common.operation.KVErrorHandler.handleRequestError(KVErrorHandler.java:115)
	at org.tikv.common.operation.KVErrorHandler.handleResponseError(KVErrorHandler.java:91)
	at org.tikv.common.policy.RetryPolicy.callWithRetry(RetryPolicy.java:87)
	at org.tikv.common.AbstractGRPCClient.callWithRetry(AbstractGRPCClient.java:85)
	at org.tikv.common.region.RegionStoreClient.coprocess(RegionStoreClient.java:612)
	at org.tikv.common.operation.iterator.DAGIterator.process(DAGIterator.java:214)
	... 7 more
Caused by: org.tikv.common.exception.GrpcException: send tikv request error: Request Failed with unknown reason for [{Region[7134] ConfVer[5] Version[715] Store[2] KeyRange[t\200\000\000\000\000\000\005%_r\223\353\361\356\346\355\223\244]:[t\200\000\000\000\000\000\005%_r\223\353\361\374<\322\000a]}], try next peer later
	at org.tikv.common.operation.RegionErrorHandler.handleRequestError(RegionErrorHandler.java:179)
	... 13 more
Caused by: org.tikv.common.exception.GrpcException: Request Failed with unknown reason for [{Region[7134] ConfVer[5] Version[715] Store[2] KeyRange[t\200\000\000\000\000\000\005%_r\223\353\361\356\346\355\223\244]:[t\200\000\000\000\000\000\005%_r\223\353\361\374<\322\000a]}]
	... 12 more