Common labels: {"app":"milvus","cluster":"4am","component":"datanode","container":"datanode","job":"qa-milvus/milvus","namespace":"qa-milvus","node_name":"4am-node32","pod":"inverted-corn-c79tm-5-49-7343-milvus-datanode-54f56c4cfb-4bwbv","stream":"stdout"} Line limit: "5000 (294 returned)" Total bytes processed: "281 kB" 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [INFO] [msgdispatcher/dispatcher.go:200] ["handle signal done"] [pchannel=by-dev-rootcoord-dml_0] [signal=terminate] [isMain=true] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [INFO] [msgdispatcher/client.go:100] ["deregister done"] [role=datanode] [nodeID=0] [vchannel=by-dev-rootcoord-dml_0_448050465291633127v0] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [DEBUG] [flowgraph/node.go:195] ["flow graph node closed"] [nodeName=dmInputNode-data-448050465291633127-by-dev-rootcoord-dml_0_448050465291633127v0] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [INFO] [datanode/flow_graph_dd_node.go:288] ["Flowgraph DD Node closing"] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [DEBUG] [flowgraph/node.go:195] ["flow graph node closed"] [nodeName=ddNode-448050465291633127-by-dev-rootcoord-dml_0_448050465291633127v0] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [DEBUG] [flowgraph/node.go:195] ["flow graph node closed"] [nodeName=BaseNode] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [DEBUG] [flowgraph/node.go:195] ["flow graph node closed"] [nodeName=ttNode-by-dev-rootcoord-dml_0_448050465291633127v0] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [INFO] [datanode/data_sync_service.go:118] ["dataSyncService flowgraph closed"] [collectionID=448050465291633127] [vChanName=by-dev-rootcoord-dml_0_448050465291633127v0] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [INFO] [datanode/data_sync_service.go:123] ["dataSyncService closed"] [collectionID=448050465291633127] [vChanName=by-dev-rootcoord-dml_0_448050465291633127v0] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [INFO] [datanode/event_manager.go:130] ["DataNode is handling watchInfo DELETE event"] [key=by-dev/meta/channelwatch/6/by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [INFO] [datanode/data_node.go:328] ["try to release flowgraph"] [vChanName=by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [INFO] [datanode/data_sync_service.go:115] ["dataSyncService closing flowgraph"] [collectionID=448050465291633127] [vChanName=by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [INFO] [msgdispatcher/dispatcher.go:177] ["get signal"] [pchannel=by-dev-rootcoord-dml_1] [signal=pause] [isMain=true] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [INFO] [msgdispatcher/dispatcher.go:210] ["stop working"] [pchannel=by-dev-rootcoord-dml_1] [isMain=true] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [INFO] [msgdispatcher/dispatcher.go:200] ["handle signal done"] [pchannel=by-dev-rootcoord-dml_1] [signal=pause] [isMain=true] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [INFO] [msgdispatcher/dispatcher.go:164] ["closed target"] [vchannel=by-dev-rootcoord-dml_1_448050465291633127v1] [isMain=true] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [INFO] [msgdispatcher/dispatcher.go:177] ["get signal"] [pchannel=by-dev-rootcoord-dml_1] [signal=terminate] [isMain=true] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [INFO] [msgstream/mq_msgstream.go:216] ["start to close mq msg stream"] ["producer num"=0] ["consumer num"=1] 2024-02-29 17:06:11.627 [2024/02/29 09:06:11.627 +00:00] [INFO] [msgdispatcher/manager.go:163] ["dispatcherManager exited"] [role=datanode] [nodeID=0] [pchannel=by-dev-rootcoord-dml_0] 2024-02-29 17:06:11.634 [2024/02/29 09:06:11.634 +00:00] [WARN] [retry/retry.go:46] ["retry func failed"] [retried=0] [error="etcdserver: requested lease not found"] 2024-02-29 17:06:11.649 [2024/02/29 09:06:11.649 +00:00] [DEBUG] [io/binlog_io.go:89] ["BinlogIO uplaod"] [paths="[file/insert_log/448050465291633127/448050465291633128/448050465291861642/103/448050465291862031,file/insert_log/448050465291633127/448050465291633128/448050465291861642/104/448050465291862032,file/insert_log/448050465291633127/448050465291633128/448050465291861642/0/448050465291862033,file/insert_log/448050465291633127/448050465291633128/448050465291861642/1/448050465291862034,file/insert_log/448050465291633127/448050465291633128/448050465291861642/100/448050465291862028,file/insert_log/448050465291633127/448050465291633128/448050465291861642/101/448050465291862029,file/insert_log/448050465291633127/448050465291633128/448050465291861642/102/448050465291862030]"] 2024-02-29 17:06:11.654 [2024/02/29 09:06:11.654 +00:00] [DEBUG] [io/binlog_io.go:89] ["BinlogIO uplaod"] [paths="[file/insert_log/448050465291633127/448050465291633128/448050465291861643/100/448050465291862035,file/insert_log/448050465291633127/448050465291633128/448050465291861643/101/448050465291862036,file/insert_log/448050465291633127/448050465291633128/448050465291861643/102/448050465291862037,file/insert_log/448050465291633127/448050465291633128/448050465291861643/103/448050465291862038,file/insert_log/448050465291633127/448050465291633128/448050465291861643/104/448050465291862039,file/insert_log/448050465291633127/448050465291633128/448050465291861643/0/448050465291862040,file/insert_log/448050465291633127/448050465291633128/448050465291861643/1/448050465291862041]"] 2024-02-29 17:06:11.655 [2024/02/29 09:06:11.655 +00:00] [INFO] [syncmgr/sync_manager.go:132] ["task calculated target segment id"] [targetID=448050465293336253] [segmentID=448050465293336253] 2024-02-29 17:06:11.655 [2024/02/29 09:06:11.655 +00:00] [INFO] [msgdispatcher/dispatcher.go:200] ["handle signal done"] [pchannel=by-dev-rootcoord-dml_1] [signal=terminate] [isMain=true] 2024-02-29 17:06:11.655 [2024/02/29 09:06:11.655 +00:00] [INFO] [msgdispatcher/client.go:100] ["deregister done"] [role=datanode] [nodeID=0] [vchannel=by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.655 [2024/02/29 09:06:11.655 +00:00] [DEBUG] [flowgraph/node.go:195] ["flow graph node closed"] [nodeName=dmInputNode-data-448050465291633127-by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.655 [2024/02/29 09:06:11.655 +00:00] [INFO] [datanode/flow_graph_dd_node.go:288] ["Flowgraph DD Node closing"] 2024-02-29 17:06:11.655 [2024/02/29 09:06:11.655 +00:00] [DEBUG] [flowgraph/node.go:195] ["flow graph node closed"] [nodeName=ddNode-448050465291633127-by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.655 [2024/02/29 09:06:11.655 +00:00] [DEBUG] [flowgraph/node.go:195] ["flow graph node closed"] [nodeName=BaseNode] 2024-02-29 17:06:11.655 [2024/02/29 09:06:11.655 +00:00] [DEBUG] [flowgraph/node.go:195] ["flow graph node closed"] [nodeName=ttNode-by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.655 [2024/02/29 09:06:11.655 +00:00] [INFO] [msgdispatcher/manager.go:163] ["dispatcherManager exited"] [role=datanode] [nodeID=0] [pchannel=by-dev-rootcoord-dml_1] 2024-02-29 17:06:11.656 [2024/02/29 09:06:11.656 +00:00] [INFO] [syncmgr/sync_manager.go:132] ["task calculated target segment id"] [targetID=448050465293336256] [segmentID=448050465293336256] 2024-02-29 17:06:11.657 [2024/02/29 09:06:11.657 +00:00] [INFO] [syncmgr/sync_manager.go:132] ["task calculated target segment id"] [targetID=448050465293336281] [segmentID=448050465293336281] 2024-02-29 17:06:11.658 [2024/02/29 09:06:11.658 +00:00] [INFO] [datanode/compactor.go:526] ["compact download deltalogs done"] [traceID=2c87117c6daea6bbe92e6accf7d2c4dc] [planID=448050465293336141] ["timeout in seconds"=900] [elapse=88.810896ms] 2024-02-29 17:06:11.658 [2024/02/29 09:06:11.658 +00:00] [INFO] [datanode/compactor.go:169] ["mergeDeltalogs end"] [planID=448050465293336141] ["number of deleted pks to compact in insert logs"=7] [elapse=226.023µs] 2024-02-29 17:06:11.658 [2024/02/29 09:06:11.658 +00:00] [DEBUG] [datanode/binlog_io.go:47] ["down load"] [path="[file/insert_log/448050465291633127/448050465291633128/448050465293336079/104/448050465291861635,file/insert_log/448050465291633127/448050465291633128/448050465293336079/0/448050465291861636,file/insert_log/448050465291633127/448050465291633128/448050465293336079/1/448050465291861637,file/insert_log/448050465291633127/448050465291633128/448050465293336079/100/448050465291861638,file/insert_log/448050465291633127/448050465291633128/448050465293336079/101/448050465291861639,file/insert_log/448050465291633127/448050465291633128/448050465293336079/102/448050465291861633,file/insert_log/448050465291633127/448050465291633128/448050465293336079/103/448050465291861634]"] 2024-02-29 17:06:11.658 [2024/02/29 09:06:11.658 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465293336079/103/448050465291861634] 2024-02-29 17:06:11.658 [2024/02/29 09:06:11.658 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465293336079/104/448050465291861635] 2024-02-29 17:06:11.658 [2024/02/29 09:06:11.658 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465293336079/0/448050465291861636] 2024-02-29 17:06:11.658 [2024/02/29 09:06:11.658 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465293336079/1/448050465291861637] 2024-02-29 17:06:11.658 [2024/02/29 09:06:11.658 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465293336079/100/448050465291861638] 2024-02-29 17:06:11.658 [2024/02/29 09:06:11.658 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465293336079/101/448050465291861639] 2024-02-29 17:06:11.659 [2024/02/29 09:06:11.659 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465293336079/102/448050465291861633] 2024-02-29 17:06:11.659 [2024/02/29 09:06:11.659 +00:00] [INFO] [writebuffer/write_buffer.go:258] ["checkpoint evaluated"] [channel=by-dev-rootcoord-dml_1_448050465291633127v1] [cpSource=syncManager] [segmentID=448050465293336253] [cpTimestamp=448055877937922049] 2024-02-29 17:06:11.659 [2024/02/29 09:06:11.659 +00:00] [DEBUG] [datanode/flow_graph_dd_node.go:213] ["DDNode receive delete messages"] [numRows=6] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.659 [2024/02/29 09:06:11.659 +00:00] [INFO] [syncmgr/sync_manager.go:132] ["task calculated target segment id"] [targetID=448050465293336348] [segmentID=448050465293336348] 2024-02-29 17:06:11.659 [2024/02/29 09:06:11.659 +00:00] [INFO] [writebuffer/write_buffer.go:258] ["checkpoint evaluated"] [channel=by-dev-rootcoord-dml_1_448050465291633127v1] [cpSource=syncManager] [segmentID=448050465293336253] [cpTimestamp=448055877937922049] 2024-02-29 17:06:11.660 [2024/02/29 09:06:11.659 +00:00] [INFO] [flowgraph/input_node.go:91] ["input node message stream closed"] [node=dmInputNode-data-448050465291633127-by-dev-rootcoord-dml_1_448050465291633127v1] [collection=448050465291633127] [closeGracefully=false] 2024-02-29 17:06:11.660 [2024/02/29 09:06:11.660 +00:00] [WARN] [datanode/flow_graph_dd_node.go:113] ["MsgStream closed"] ["ddNode node"=ddNode-448050465291633127-by-dev-rootcoord-dml_1_448050465291633127v1] [collection=448050465291633127] [channel=by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.660 [2024/02/29 09:06:11.660 +00:00] [INFO] [datanode/data_sync_service.go:118] ["dataSyncService flowgraph closed"] [collectionID=448050465291633127] [vChanName=by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.660 [2024/02/29 09:06:11.660 +00:00] [INFO] [datanode/data_sync_service.go:123] ["dataSyncService closed"] [collectionID=448050465291633127] [vChanName=by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.661 [2024/02/29 09:06:11.661 +00:00] [INFO] [syncmgr/meta_writer.go:70] [SaveBinlogPath] [SegmentID=448050465293336256] [CollectionID=448050465291633127] [ParitionID=448050465291633128] [startPos="[{\"start_position\":{\"channel_name\":\"by-dev-rootcoord-dml_1_448050465291633127v1\",\"msgID\":\"CCAQkhsYACAA\",\"msgGroup\":\"datanode-0-by-dev-rootcoord-dml_1_448050465291633127v1-true\",\"timestamp\":448055878462210049},\"segmentID\":448050465293336397},{\"start_position\":{\"channel_name\":\"by-dev-rootcoord-dml_1_448050465291633127v1\",\"msgID\":\"CCAQkhsYACAA\",\"msgGroup\":\"datanode-0-by-dev-rootcoord-dml_1_448050465291633127v1-true\",\"timestamp\":448055878462210049},\"segmentID\":448050465293336400}]"] [checkPoints="[{\"segmentID\":448050465293336256,\"position\":{\"channel_name\":\"by-dev-rootcoord-dml_1_448050465291633127v1\",\"msgID\":\"CCAQ5xwYACAA\",\"msgGroup\":\"datanode-0-by-dev-rootcoord-dml_1_448050465291633127v1-true\",\"timestamp\":448055885592526849},\"num_of_rows\":5}]"] [binlogNum=0] [statslogNum=0] [deltalogNum=1] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.662 [2024/02/29 09:06:11.662 +00:00] [INFO] [syncmgr/meta_writer.go:70] [SaveBinlogPath] [SegmentID=448050465293336253] [CollectionID=448050465291633127] [ParitionID=448050465291633128] [startPos="[{\"start_position\":{\"channel_name\":\"by-dev-rootcoord-dml_1_448050465291633127v1\",\"msgID\":\"CCAQkhsYACAA\",\"msgGroup\":\"datanode-0-by-dev-rootcoord-dml_1_448050465291633127v1-true\",\"timestamp\":448055878462210049},\"segmentID\":448050465293336397},{\"start_position\":{\"channel_name\":\"by-dev-rootcoord-dml_1_448050465291633127v1\",\"msgID\":\"CCAQkhsYACAA\",\"msgGroup\":\"datanode-0-by-dev-rootcoord-dml_1_448050465291633127v1-true\",\"timestamp\":448055878462210049},\"segmentID\":448050465293336400}]"] [checkPoints="[{\"segmentID\":448050465293336253,\"position\":{\"channel_name\":\"by-dev-rootcoord-dml_1_448050465291633127v1\",\"msgID\":\"CCAQ5xwYACAA\",\"msgGroup\":\"datanode-0-by-dev-rootcoord-dml_1_448050465291633127v1-true\",\"timestamp\":448055885592526849},\"num_of_rows\":7}]"] [binlogNum=0] [statslogNum=0] [deltalogNum=1] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.662 [2024/02/29 09:06:11.662 +00:00] [WARN] [broker/datacoord.go:118] ["failed to SaveBinlogPaths"] [error="for node 6: channel not found[channel=by-dev-rootcoord-dml_1_448050465291633127v1]"] 2024-02-29 17:06:11.663 [2024/02/29 09:06:11.662 +00:00] [WARN] [syncmgr/meta_writer.go:117] ["meta error found, skip sync and start to drop virtual channel"] [channel=by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.663 [2024/02/29 09:06:11.662 +00:00] [INFO] [syncmgr/task.go:213] ["task done"] [collectionID=448050465291633127] [partitionID=448050465291633128] [segmentID=448050465293336256] [channel=by-dev-rootcoord-dml_1_448050465291633127v1] [level=L1] [flushedSize=638] 2024-02-29 17:06:11.663 [2024/02/29 09:06:11.663 +00:00] [WARN] [broker/datacoord.go:118] ["failed to SaveBinlogPaths"] [error="for node 6: channel not found[channel=by-dev-rootcoord-dml_1_448050465291633127v1]"] 2024-02-29 17:06:11.663 [2024/02/29 09:06:11.663 +00:00] [WARN] [syncmgr/meta_writer.go:117] ["meta error found, skip sync and start to drop virtual channel"] [channel=by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.663 [2024/02/29 09:06:11.663 +00:00] [INFO] [syncmgr/task.go:213] ["task done"] [collectionID=448050465291633127] [partitionID=448050465291633128] [segmentID=448050465293336253] [channel=by-dev-rootcoord-dml_1_448050465291633127v1] [level=L1] [flushedSize=626] 2024-02-29 17:06:11.663 [2024/02/29 09:06:11.663 +00:00] [INFO] [syncmgr/meta_writer.go:70] [SaveBinlogPath] [SegmentID=448050465293336281] [CollectionID=448050465291633127] [ParitionID=448050465291633128] [startPos="[]"] [checkPoints="[{\"segmentID\":448050465293336281,\"position\":{\"channel_name\":\"by-dev-rootcoord-dml_1_448050465291633127v1\",\"msgID\":\"CCAQ5xwYACAA\",\"msgGroup\":\"datanode-0-by-dev-rootcoord-dml_1_448050465291633127v1-true\",\"timestamp\":448055885592526849},\"num_of_rows\":7}]"] [binlogNum=0] [statslogNum=0] [deltalogNum=1] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.664 [2024/02/29 09:06:11.664 +00:00] [INFO] [syncmgr/meta_writer.go:70] [SaveBinlogPath] [SegmentID=448050465293336348] [CollectionID=448050465291633127] [ParitionID=448050465291633128] [startPos="[]"] [checkPoints="[{\"segmentID\":448050465293336348,\"position\":{\"channel_name\":\"by-dev-rootcoord-dml_1_448050465291633127v1\",\"msgID\":\"CCAQ5xwYACAA\",\"msgGroup\":\"datanode-0-by-dev-rootcoord-dml_1_448050465291633127v1-true\",\"timestamp\":448055885592526849},\"num_of_rows\":7}]"] [binlogNum=0] [statslogNum=0] [deltalogNum=1] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.664 [2024/02/29 09:06:11.664 +00:00] [WARN] [broker/datacoord.go:118] ["failed to SaveBinlogPaths"] [error="for node 6: channel not found[channel=by-dev-rootcoord-dml_1_448050465291633127v1]"] 2024-02-29 17:06:11.664 [2024/02/29 09:06:11.664 +00:00] [WARN] [syncmgr/meta_writer.go:117] ["meta error found, skip sync and start to drop virtual channel"] [channel=by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.664 [2024/02/29 09:06:11.664 +00:00] [INFO] [syncmgr/task.go:213] ["task done"] [collectionID=448050465291633127] [partitionID=448050465291633128] [segmentID=448050465293336281] [channel=by-dev-rootcoord-dml_1_448050465291633127v1] [level=L1] [flushedSize=645] 2024-02-29 17:06:11.664 [2024/02/29 09:06:11.664 +00:00] [WARN] [broker/datacoord.go:118] ["failed to SaveBinlogPaths"] [error="for node 6: channel not found[channel=by-dev-rootcoord-dml_1_448050465291633127v1]"] 2024-02-29 17:06:11.664 [2024/02/29 09:06:11.664 +00:00] [WARN] [syncmgr/meta_writer.go:117] ["meta error found, skip sync and start to drop virtual channel"] [channel=by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:11.664 [2024/02/29 09:06:11.664 +00:00] [INFO] [syncmgr/task.go:213] ["task done"] [collectionID=448050465291633127] [partitionID=448050465291633128] [segmentID=448050465293336348] [channel=by-dev-rootcoord-dml_1_448050465291633127v1] [level=L1] [flushedSize=643] 2024-02-29 17:06:11.666 [2024/02/29 09:06:11.666 +00:00] [INFO] [datanode/compactor.go:428] ["compact merge end"] [planID=448050465293336141] ["remaining insert numRows"=0] ["expired entities"=0] ["binlog file number"=0] ["download insert log elapse"=8.007675ms] ["upload insert log elapse"=0s] ["merge elapse"=8.476063ms] 2024-02-29 17:06:11.666 [2024/02/29 09:06:11.666 +00:00] [INFO] [datanode/compactor.go:552] ["compact done"] [traceID=2c87117c6daea6bbe92e6accf7d2c4dc] [planID=448050465293336141] ["timeout in seconds"=900] [targetSegmentID=448050465291862027] [compactedFrom="[448050465293336079]"] ["num of binlog paths"=0] ["num of stats paths"=0] ["num of delta paths"=0] [elapse=97.629973ms] 2024-02-29 17:06:11.666 [2024/02/29 09:06:11.666 +00:00] [INFO] [datanode/compaction_executor.go:113] ["end to execute compaction"] [planID=448050465293336141] [Collection=448050465291633127] [channel=by-dev-rootcoord-dml_1_448050465291633127v1] [planID=448050465293336141] 2024-02-29 17:06:11.954 [2024/02/29 09:06:11.953 +00:00] [DEBUG] [datanode/binlog_io.go:47] ["down load"] [path="[file/insert_log/448050465291633127/448050465291633128/448050465291860113/103/448050465291860201,file/insert_log/448050465291633127/448050465291633128/448050465291860113/104/448050465291860202,file/insert_log/448050465291633127/448050465291633128/448050465291860113/0/448050465291860203,file/insert_log/448050465291633127/448050465291633128/448050465291860113/1/448050465291860204,file/insert_log/448050465291633127/448050465291633128/448050465291860113/100/448050465291860198,file/insert_log/448050465291633127/448050465291633128/448050465291860113/101/448050465291860199,file/insert_log/448050465291633127/448050465291633128/448050465291860113/102/448050465291860200]"] 2024-02-29 17:06:11.954 [2024/02/29 09:06:11.953 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291860113/102/448050465291860200] 2024-02-29 17:06:11.954 [2024/02/29 09:06:11.953 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291860113/103/448050465291860201] 2024-02-29 17:06:11.954 [2024/02/29 09:06:11.954 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291860113/1/448050465291860204] 2024-02-29 17:06:11.954 [2024/02/29 09:06:11.954 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291860113/101/448050465291860199] 2024-02-29 17:06:11.954 [2024/02/29 09:06:11.954 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291860113/104/448050465291860202] 2024-02-29 17:06:11.954 [2024/02/29 09:06:11.954 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291860113/100/448050465291860198] 2024-02-29 17:06:11.954 [2024/02/29 09:06:11.954 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291860113/0/448050465291860203] 2024-02-29 17:06:11.969 [2024/02/29 09:06:11.968 +00:00] [DEBUG] [datanode/binlog_io.go:47] ["down load"] [path="[file/insert_log/448050465291633127/448050465291633128/448050465291858599/101/448050465291858685,file/insert_log/448050465291633127/448050465291633128/448050465291858599/102/448050465291858686,file/insert_log/448050465291633127/448050465291633128/448050465291858599/103/448050465291858687,file/insert_log/448050465291633127/448050465291633128/448050465291858599/104/448050465291858688,file/insert_log/448050465291633127/448050465291633128/448050465291858599/0/448050465291858689,file/insert_log/448050465291633127/448050465291633128/448050465291858599/1/448050465291858690,file/insert_log/448050465291633127/448050465291633128/448050465291858599/100/448050465291858684]"] 2024-02-29 17:06:11.969 [2024/02/29 09:06:11.969 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291858599/100/448050465291858684] 2024-02-29 17:06:11.969 [2024/02/29 09:06:11.969 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291858599/101/448050465291858685] 2024-02-29 17:06:11.969 [2024/02/29 09:06:11.969 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291858599/102/448050465291858686] 2024-02-29 17:06:11.969 [2024/02/29 09:06:11.969 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291858599/103/448050465291858687] 2024-02-29 17:06:11.969 [2024/02/29 09:06:11.969 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291858599/104/448050465291858688] 2024-02-29 17:06:11.969 [2024/02/29 09:06:11.969 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291858599/0/448050465291858689] 2024-02-29 17:06:11.969 [2024/02/29 09:06:11.969 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291858599/1/448050465291858690] 2024-02-29 17:06:12.625 [2024/02/29 09:06:12.625 +00:00] [DEBUG] [io/binlog_io.go:89] ["BinlogIO uplaod"] [paths="[file/insert_log/448050465291633127/448050465291633128/448050465291861643/1/448050465291862052,file/insert_log/448050465291633127/448050465291633128/448050465291861643/100/448050465291862046,file/insert_log/448050465291633127/448050465291633128/448050465291861643/101/448050465291862047,file/insert_log/448050465291633127/448050465291633128/448050465291861643/102/448050465291862048,file/insert_log/448050465291633127/448050465291633128/448050465291861643/103/448050465291862049,file/insert_log/448050465291633127/448050465291633128/448050465291861643/104/448050465291862050,file/insert_log/448050465291633127/448050465291633128/448050465291861643/0/448050465291862051]"] 2024-02-29 17:06:12.688 [2024/02/29 09:06:12.688 +00:00] [DEBUG] [io/binlog_io.go:89] ["BinlogIO uplaod"] [paths="[file/insert_log/448050465291633127/448050465291633128/448050465291861642/102/448050465291862055,file/insert_log/448050465291633127/448050465291633128/448050465291861642/103/448050465291862056,file/insert_log/448050465291633127/448050465291633128/448050465291861642/104/448050465291862057,file/insert_log/448050465291633127/448050465291633128/448050465291861642/0/448050465291862058,file/insert_log/448050465291633127/448050465291633128/448050465291861642/1/448050465291862059,file/insert_log/448050465291633127/448050465291633128/448050465291861642/100/448050465291862053,file/insert_log/448050465291633127/448050465291633128/448050465291861642/101/448050465291862054]"] 2024-02-29 17:06:12.953 [2024/02/29 09:06:12.953 +00:00] [DEBUG] [datanode/binlog_io.go:47] ["down load"] [path="[file/insert_log/448050465291633127/448050465291633128/448050465291860113/103/448050465291860208,file/insert_log/448050465291633127/448050465291633128/448050465291860113/104/448050465291860209,file/insert_log/448050465291633127/448050465291633128/448050465291860113/0/448050465291860210,file/insert_log/448050465291633127/448050465291633128/448050465291860113/1/448050465291860211,file/insert_log/448050465291633127/448050465291633128/448050465291860113/100/448050465291860205,file/insert_log/448050465291633127/448050465291633128/448050465291860113/101/448050465291860206,file/insert_log/448050465291633127/448050465291633128/448050465291860113/102/448050465291860207]"] 2024-02-29 17:06:12.953 [2024/02/29 09:06:12.953 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291860113/102/448050465291860207] 2024-02-29 17:06:12.953 [2024/02/29 09:06:12.953 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291860113/103/448050465291860208] 2024-02-29 17:06:12.953 [2024/02/29 09:06:12.953 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291860113/1/448050465291860211] 2024-02-29 17:06:12.953 [2024/02/29 09:06:12.953 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291860113/100/448050465291860205] 2024-02-29 17:06:12.953 [2024/02/29 09:06:12.953 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291860113/104/448050465291860209] 2024-02-29 17:06:12.953 [2024/02/29 09:06:12.953 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291860113/101/448050465291860206] 2024-02-29 17:06:12.953 [2024/02/29 09:06:12.953 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291860113/0/448050465291860210] 2024-02-29 17:06:12.975 [2024/02/29 09:06:12.975 +00:00] [DEBUG] [datanode/binlog_io.go:47] ["down load"] [path="[file/insert_log/448050465291633127/448050465291633128/448050465291858599/101/448050465291858692,file/insert_log/448050465291633127/448050465291633128/448050465291858599/102/448050465291858693,file/insert_log/448050465291633127/448050465291633128/448050465291858599/103/448050465291858694,file/insert_log/448050465291633127/448050465291633128/448050465291858599/104/448050465291858695,file/insert_log/448050465291633127/448050465291633128/448050465291858599/0/448050465291858696,file/insert_log/448050465291633127/448050465291633128/448050465291858599/1/448050465291858697,file/insert_log/448050465291633127/448050465291633128/448050465291858599/100/448050465291858691]"] 2024-02-29 17:06:12.988 [2024/02/29 09:06:12.975 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291858599/100/448050465291858691] 2024-02-29 17:06:12.988 [2024/02/29 09:06:12.976 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291858599/101/448050465291858692] 2024-02-29 17:06:12.988 [2024/02/29 09:06:12.976 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291858599/102/448050465291858693] 2024-02-29 17:06:12.988 [2024/02/29 09:06:12.976 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291858599/103/448050465291858694] 2024-02-29 17:06:12.988 [2024/02/29 09:06:12.977 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291858599/104/448050465291858695] 2024-02-29 17:06:12.988 [2024/02/29 09:06:12.977 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291858599/0/448050465291858696] 2024-02-29 17:06:12.988 [2024/02/29 09:06:12.977 +00:00] [DEBUG] [io/binlog_io.go:61] ["BinlogIO download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291858599/1/448050465291858697] 2024-02-29 17:06:13.000 [2024/02/29 09:06:13.000 +00:00] [DEBUG] [datanode/services.go:73] ["DataNode current state"] [State=Healthy] 2024-02-29 17:06:13.093 [2024/02/29 09:06:13.092 +00:00] [WARN] [sessionutil/session_util.go:550] ["fail to retry keepAliveOnce"] [serverName=datanode] [LeaseID=2400574675635401385] [error="etcdserver: requested lease not found"] 2024-02-29 17:06:13.094 [2024/02/29 09:06:13.093 +00:00] [WARN] [sessionutil/session_util.go:878] ["connection lost detected, shuting down"] 2024-02-29 17:06:13.094 [2024/02/29 09:06:13.093 +00:00] [ERROR] [datanode/data_node.go:201] ["Data Node disconnected from etcd, process will exit"] ["Server Id"=6] [stack="github.com/milvus-io/milvus/internal/datanode.(*DataNode).Register.func1\n\t/go/src/github.com/milvus-io/milvus/internal/datanode/data_node.go:201"] 2024-02-29 17:06:13.094 [2024/02/29 09:06:13.094 +00:00] [INFO] [writebuffer/manager.go:83] ["buffer manager memory check stopped"] 2024-02-29 17:06:13.094 [2024/02/29 09:06:13.094 +00:00] [INFO] [datanode/data_node.go:449] ["close id allocator"] [role=datanode] 2024-02-29 17:06:13.095 [2024/02/29 09:06:13.095 +00:00] [INFO] [datanode/event_manager.go:63] ["watch etcd loop quit"] 2024-02-29 17:06:13.095 [2024/02/29 09:06:13.095 +00:00] [WARN] [datanode/data_node.go:342] ["DataNode context done, exiting background GC"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [storage/remote_chunk_manager.go:190] ["failed to read object"] [bucket=milvus-bucket] [path=file/insert_log/448050465291633127/448050465291633128/448050465291860113/104/448050465291860209] [error="IO failed[key=file/insert_log/448050465291633127/448050465291633128/448050465291860113/104/448050465291860209]: context canceled"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [retry/retry.go:46] ["retry func failed"] [traceID=48e051563eacdde9ce581c19a13e488e] [retried=0] [error="IO failed[key=file/insert_log/448050465291633127/448050465291633128/448050465291860113/104/448050465291860209]: context canceled"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [storage/remote_chunk_manager.go:190] ["failed to read object"] [bucket=milvus-bucket] [path=file/insert_log/448050465291633127/448050465291633128/448050465291858599/104/448050465291858695] [error="IO failed[key=file/insert_log/448050465291633127/448050465291633128/448050465291858599/104/448050465291858695]: context canceled"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [retry/retry.go:46] ["retry func failed"] [traceID=7a7b245d841aee3c1f73512520202ca4] [retried=0] [error="IO failed[key=file/insert_log/448050465291633127/448050465291633128/448050465291858599/104/448050465291858695]: context canceled"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [io/binlog_io.go:65] ["BinlogIO fail to download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291858599/104/448050465291858695] [error="IO failed[key=file/insert_log/448050465291633127/448050465291633128/448050465291858599/104/448050465291858695]: context canceled"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [retry/retry.go:46] ["retry func failed"] [traceID=7a7b245d841aee3c1f73512520202ca4] [retried=0] [error="IO failed[key=file/insert_log/448050465291633127/448050465291633128/448050465291858599/104/448050465291858695]: context canceled"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [datanode/binlog_io.go:50] ["ctx done when downloading kvs from blob storage"] [paths="[file/insert_log/448050465291633127/448050465291633128/448050465291858599/101/448050465291858692,file/insert_log/448050465291633127/448050465291633128/448050465291858599/102/448050465291858693,file/insert_log/448050465291633127/448050465291633128/448050465291858599/103/448050465291858694,file/insert_log/448050465291633127/448050465291633128/448050465291858599/104/448050465291858695,file/insert_log/448050465291633127/448050465291633128/448050465291858599/0/448050465291858696,file/insert_log/448050465291633127/448050465291633128/448050465291858599/1/448050465291858697,file/insert_log/448050465291633127/448050465291633128/448050465291858599/100/448050465291858691]"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [io/binlog_io.go:65] ["BinlogIO fail to download"] [path=file/insert_log/448050465291633127/448050465291633128/448050465291860113/104/448050465291860209] [error="IO failed[key=file/insert_log/448050465291633127/448050465291633128/448050465291860113/104/448050465291860209]: context canceled"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [retry/retry.go:46] ["retry func failed"] [traceID=48e051563eacdde9ce581c19a13e488e] [retried=0] [error="IO failed[key=file/insert_log/448050465291633127/448050465291633128/448050465291860113/104/448050465291860209]: context canceled"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [datanode/binlog_io.go:50] ["ctx done when downloading kvs from blob storage"] [paths="[file/insert_log/448050465291633127/448050465291633128/448050465291860113/103/448050465291860208,file/insert_log/448050465291633127/448050465291633128/448050465291860113/104/448050465291860209,file/insert_log/448050465291633127/448050465291633128/448050465291860113/0/448050465291860210,file/insert_log/448050465291633127/448050465291633128/448050465291860113/1/448050465291860211,file/insert_log/448050465291633127/448050465291633128/448050465291860113/100/448050465291860205,file/insert_log/448050465291633127/448050465291633128/448050465291860113/101/448050465291860206,file/insert_log/448050465291633127/448050465291633128/448050465291860113/102/448050465291860207]"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [datanode/compactor.go:331] ["download insertlogs wrong"] [planID=448050465293336110] [path="[file/insert_log/448050465291633127/448050465291633128/448050465291858599/101/448050465291858692,file/insert_log/448050465291633127/448050465291633128/448050465291858599/102/448050465291858693,file/insert_log/448050465291633127/448050465291633128/448050465291858599/103/448050465291858694,file/insert_log/448050465291633127/448050465291633128/448050465291858599/104/448050465291858695,file/insert_log/448050465291633127/448050465291633128/448050465291858599/0/448050465291858696,file/insert_log/448050465291633127/448050465291633128/448050465291858599/1/448050465291858697,file/insert_log/448050465291633127/448050465291633128/448050465291858599/100/448050465291858691]"] [error="download from blob storage wrong"] [errorVerbose="download from blob storage wrong\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/internal/datanode.init\n | \t/go/src/github.com/milvus-io/milvus/internal/datanode/binlog_io.go:39\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6525\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.main\n | \t/usr/local/go/src/runtime/proc.go:233\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) download from blob storage wrong\nError types: (1) *withstack.withStack (2) *errutil.leafError"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [datanode/compactor.go:331] ["download insertlogs wrong"] [planID=448050465293336111] [path="[file/insert_log/448050465291633127/448050465291633128/448050465291860113/103/448050465291860208,file/insert_log/448050465291633127/448050465291633128/448050465291860113/104/448050465291860209,file/insert_log/448050465291633127/448050465291633128/448050465291860113/0/448050465291860210,file/insert_log/448050465291633127/448050465291633128/448050465291860113/1/448050465291860211,file/insert_log/448050465291633127/448050465291633128/448050465291860113/100/448050465291860205,file/insert_log/448050465291633127/448050465291633128/448050465291860113/101/448050465291860206,file/insert_log/448050465291633127/448050465291633128/448050465291860113/102/448050465291860207]"] [error="download from blob storage wrong"] [errorVerbose="download from blob storage wrong\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/internal/datanode.init\n | \t/go/src/github.com/milvus-io/milvus/internal/datanode/binlog_io.go:39\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6525\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.main\n | \t/usr/local/go/src/runtime/proc.go:233\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) download from blob storage wrong\nError types: (1) *withstack.withStack (2) *errutil.leafError"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [datanode/compactor.go:540] ["compact wrong, fail to merge"] [traceID=7a7b245d841aee3c1f73512520202ca4] [planID=448050465293336110] ["timeout in seconds"=900] [error="download from blob storage wrong"] [errorVerbose="download from blob storage wrong\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/internal/datanode.init\n | \t/go/src/github.com/milvus-io/milvus/internal/datanode/binlog_io.go:39\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6525\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.main\n | \t/usr/local/go/src/runtime/proc.go:233\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) download from blob storage wrong\nError types: (1) *withstack.withStack (2) *errutil.leafError"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [datanode/compactor.go:540] ["compact wrong, fail to merge"] [traceID=48e051563eacdde9ce581c19a13e488e] [planID=448050465293336111] ["timeout in seconds"=900] [error="download from blob storage wrong"] [errorVerbose="download from blob storage wrong\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/internal/datanode.init\n | \t/go/src/github.com/milvus-io/milvus/internal/datanode/binlog_io.go:39\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6525\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.main\n | \t/usr/local/go/src/runtime/proc.go:233\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) download from blob storage wrong\nError types: (1) *withstack.withStack (2) *errutil.leafError"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [datanode/compaction_executor.go:107] ["compaction task failed"] [planID=448050465293336110] [Collection=448050465291633127] [channel=by-dev-rootcoord-dml_1_448050465291633127v1] [error="download from blob storage wrong"] [errorVerbose="download from blob storage wrong\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/internal/datanode.init\n | \t/go/src/github.com/milvus-io/milvus/internal/datanode/binlog_io.go:39\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6525\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.main\n | \t/usr/local/go/src/runtime/proc.go:233\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) download from blob storage wrong\nError types: (1) *withstack.withStack (2) *errutil.leafError"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [INFO] [datanode/compaction_executor.go:113] ["end to execute compaction"] [planID=448050465293336110] [Collection=448050465291633127] [channel=by-dev-rootcoord-dml_1_448050465291633127v1] [planID=448050465293336110] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [datanode/compaction_executor.go:107] ["compaction task failed"] [planID=448050465293336111] [Collection=448050465291633127] [channel=by-dev-rootcoord-dml_0_448050465291633127v0] [error="download from blob storage wrong"] [errorVerbose="download from blob storage wrong\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/internal/datanode.init\n | \t/go/src/github.com/milvus-io/milvus/internal/datanode/binlog_io.go:39\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6525\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.doInit\n | \t/usr/local/go/src/runtime/proc.go:6502\n | runtime.main\n | \t/usr/local/go/src/runtime/proc.go:233\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) download from blob storage wrong\nError types: (1) *withstack.withStack (2) *errutil.leafError"] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [INFO] [datanode/compaction_executor.go:113] ["end to execute compaction"] [planID=448050465293336111] [Collection=448050465291633127] [channel=by-dev-rootcoord-dml_0_448050465291633127v0] [planID=448050465293336111] 2024-02-29 17:06:13.096 [2024/02/29 09:06:13.096 +00:00] [WARN] [syncmgr/task.go:167] ["failed allocate ids for sync task"] [collectionID=448050465291633127] [partitionID=448050465291633128] [segmentID=448050465291858599] [channel=by-dev-rootcoord-dml_1_448050465291633127v1] [level=L1] [error="fail to allocate ID, closed allocator"] [errorVerbose="fail to allocate ID, closed allocator\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/internal/allocator.(*IDAllocator).Alloc\n | \t/go/src/github.com/milvus-io/milvus/internal/allocator/id_allocator.go:151\n | github.com/milvus-io/milvus/internal/datanode/syncmgr.(*SyncTask).prefetchIDs\n | \t/go/src/github.com/milvus-io/milvus/internal/datanode/syncmgr/task.go:231\n | github.com/milvus-io/milvus/internal/datanode/syncmgr.(*SyncTask).Run\n | \t/go/src/github.com/milvus-io/milvus/internal/datanode/syncmgr/task.go:165\n | github.com/milvus-io/milvus/internal/datanode/syncmgr.(*keyLockDispatcher[...]).Submit.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/datanode/syncmgr/key_lock_dispatcher.go:36\n | github.com/milvus-io/milvus/pkg/util/conc.(*Pool[...]).Submit.func1\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/conc/pool.go:81\n | github.com/panjf2000/ants/v2.(*goWorker).run.func1\n | \t/go/pkg/mod/github.com/panjf2000/ants/v2@v2.7.2/worker.go:67\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) fail to allocate ID, closed allocator\nError types: (1) *withstack.withStack (2) *errutil.leafError"] 2024-02-29 17:06:13.097 [2024/02/29 09:06:13.096 +00:00] [ERROR] [conc/options.go:54] ["Conc pool panicked"] [panic="fail to allocate ID, closed allocator"] [panicVerbose="fail to allocate ID, closed allocator\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/internal/allocator.(*IDAllocator).Alloc\n | \t/go/src/github.com/milvus-io/milvus/internal/allocator/id_allocator.go:151\n | github.com/milvus-io/milvus/internal/datanode/syncmgr.(*SyncTask).prefetchIDs\n | \t/go/src/github.com/milvus-io/milvus/internal/datanode/syncmgr/task.go:231\n | github.com/milvus-io/milvus/internal/datanode/syncmgr.(*SyncTask).Run\n | \t/go/src/github.com/milvus-io/milvus/internal/datanode/syncmgr/task.go:165\n | github.com/milvus-io/milvus/internal/datanode/syncmgr.(*keyLockDispatcher[...]).Submit.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/datanode/syncmgr/key_lock_dispatcher.go:36\n | github.com/milvus-io/milvus/pkg/util/conc.(*Pool[...]).Submit.func1\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/conc/pool.go:81\n | github.com/panjf2000/ants/v2.(*goWorker).run.func1\n | \t/go/pkg/mod/github.com/panjf2000/ants/v2@v2.7.2/worker.go:67\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) fail to allocate ID, closed allocator\nError types: (1) *withstack.withStack (2) *errutil.leafError"] [stack="github.com/milvus-io/milvus/pkg/util/conc.(*poolOption).antsOptions.func1\n\t/go/src/github.com/milvus-io/milvus/pkg/util/conc/options.go:54\ngithub.com/panjf2000/ants/v2.(*goWorker).run.func1.1\n\t/go/pkg/mod/github.com/panjf2000/ants/v2@v2.7.2/worker.go:54\nruntime.gopanic\n\t/usr/local/go/src/runtime/panic.go:884\ngithub.com/milvus-io/milvus/pkg/util/conc.(*Pool[...]).Submit.func1.1\n\t/go/src/github.com/milvus-io/milvus/pkg/util/conc/pool.go:74\nruntime.gopanic\n\t/usr/local/go/src/runtime/panic.go:884\ngithub.com/milvus-io/milvus/internal/datanode/syncmgr.(*storageV1Serializer).setTaskMeta.func1\n\t/go/src/github.com/milvus-io/milvus/internal/datanode/syncmgr/storage_serializer.go:156\ngithub.com/milvus-io/milvus/internal/datanode/syncmgr.(*SyncTask).handleError\n\t/go/src/github.com/milvus-io/milvus/internal/datanode/syncmgr/task.go:117\ngithub.com/milvus-io/milvus/internal/datanode/syncmgr.(*SyncTask).Run.func1\n\t/go/src/github.com/milvus-io/milvus/internal/datanode/syncmgr/task.go:132\ngithub.com/milvus-io/milvus/internal/datanode/syncmgr.(*SyncTask).Run\n\t/go/src/github.com/milvus-io/milvus/internal/datanode/syncmgr/task.go:168\ngithub.com/milvus-io/milvus/internal/datanode/syncmgr.(*keyLockDispatcher[...]).Submit.func1\n\t/go/src/github.com/milvus-io/milvus/internal/datanode/syncmgr/key_lock_dispatcher.go:36\ngithub.com/milvus-io/milvus/pkg/util/conc.(*Pool[...]).Submit.func1\n\t/go/src/github.com/milvus-io/milvus/pkg/util/conc/pool.go:81\ngithub.com/panjf2000/ants/v2.(*goWorker).run.func1\n\t/go/pkg/mod/github.com/panjf2000/ants/v2@v2.7.2/worker.go:67"] 2024-02-29 17:06:50.434 [2024/02/29 09:06:50.434 +00:00] [INFO] [roles/roles.go:304] ["starting running Milvus components"] 2024-02-29 17:06:50.434 [2024/02/29 09:06:50.434 +00:00] [INFO] [roles/roles.go:167] ["Enable Jemalloc"] ["Jemalloc Path"=/milvus/lib/libjemalloc.so] 2024-02-29 17:06:50.445 [2024/02/29 09:06:50.445 +00:00] [DEBUG] [config/refresher.go:67] ["start refreshing configurations"] [source=FileSource] 2024-02-29 17:06:50.445 [2024/02/29 09:06:50.445 +00:00] [DEBUG] [config/etcd_source.go:50] ["init etcd source"] [etcdInfo="{\"UseEmbed\":false,\"UseSSL\":false,\"Endpoints\":[\"inverted-corn-c79tm-5-49-7343-etcd:2379\"],\"KeyPrefix\":\"by-dev\",\"CertFile\":\"/path/to/etcd-client.pem\",\"KeyFile\":\"/path/to/etcd-client-key.pem\",\"CaCertFile\":\"/path/to/ca.pem\",\"MinVersion\":\"1.3\",\"RefreshInterval\":5000000000}"] 2024-02-29 17:06:50.445 [2024/02/29 09:06:50.445 +00:00] [INFO] [etcd/etcd_util.go:47] ["create etcd client"] [useEmbedEtcd=false] [useSSL=false] [endpoints="[inverted-corn-c79tm-5-49-7343-etcd:2379]"] [minVersion=1.3] 2024-02-29 17:06:50.446 [2024/02/29 09:06:50.446 +00:00] [DEBUG] [config/etcd_source.go:86] ["etcd refreshConfigurations"] [prefix=by-dev/config] [endpoints="[inverted-corn-c79tm-5-49-7343-etcd:2379]"] 2024-02-29 17:06:50.448 [2024/02/29 09:06:50.448 +00:00] [DEBUG] [config/refresher.go:67] ["start refreshing configurations"] [source=EtcdSource] 2024-02-29 17:06:50.449 [2024/02/29 09:06:50.449 +00:00] [DEBUG] [config/refresher.go:67] ["start refreshing configurations"] [source=FileSource] 2024-02-29 17:06:50.449 [2024/02/29 09:06:50.449 +00:00] [INFO] [paramtable/hook_config.go:21] ["hook config"] [hook={}] 2024-02-29 17:06:50.449 [2024/02/29 09:06:50.449 +00:00] [INFO] [roles/roles.go:253] [setupPrometheusHTTPServer] 2024-02-29 17:06:50.449 [2024/02/29 09:06:50.449 +00:00] [INFO] [http/server.go:112] ["management listen"] [addr=:9091] 2024-02-29 17:06:50.449 [2024/02/29 09:06:50.449 +00:00] [INFO] [etcd/etcd_util.go:47] ["create etcd client"] [useEmbedEtcd=false] [useSSL=false] [endpoints="[inverted-corn-c79tm-5-49-7343-etcd:2379]"] [minVersion=1.3] 2024-02-29 17:06:50.449 [2024/02/29 09:06:50.449 +00:00] [INFO] [datanode/service.go:249] ["DataNode address"] [address=10.104.26.144:21124] 2024-02-29 17:06:50.449 [2024/02/29 09:06:50.449 +00:00] [INFO] [datanode/service.go:250] ["DataNode serverID"] [serverID=0] 2024-02-29 17:06:50.550 [2024/02/29 09:06:50.550 +00:00] [INFO] [datanode/service.go:259] ["initializing RootCoord client for DataNode"] 2024-02-29 17:06:50.550 [2024/02/29 09:06:50.550 +00:00] [INFO] [etcd/etcd_util.go:47] ["create etcd client"] [useEmbedEtcd=false] [useSSL=false] [endpoints="[inverted-corn-c79tm-5-49-7343-etcd:2379]"] [minVersion=1.3] 2024-02-29 17:06:50.550 [2024/02/29 09:06:50.550 +00:00] [DEBUG] [sessionutil/session_util.go:254] ["Session try to connect to etcd"] 2024-02-29 17:06:50.553 [2024/02/29 09:06:50.553 +00:00] [DEBUG] [sessionutil/session_util.go:269] ["Session connect to etcd success"] 2024-02-29 17:06:50.554 [2024/02/29 09:06:50.554 +00:00] [DEBUG] [sessionutil/session_util.go:617] ["SessionUtil GetSessions"] [prefix=rootcoord] [key=rootcoord] [address=10.104.23.167:53100] 2024-02-29 17:06:50.554 [2024/02/29 09:06:50.554 +00:00] [DEBUG] [client/client.go:93] ["RootCoordClient GetSessions success"] [address=10.104.23.167:53100] [serverID=5] 2024-02-29 17:06:50.556 [2024/02/29 09:06:50.556 +00:00] [INFO] [componentutil/componentutil.go:61] ["WaitForComponentStates success"] ["current state"=Healthy] 2024-02-29 17:06:50.556 [2024/02/29 09:06:50.556 +00:00] [INFO] [datanode/service.go:270] ["RootCoord client is ready for DataNode"] 2024-02-29 17:06:50.556 [2024/02/29 09:06:50.556 +00:00] [DEBUG] [datanode/service.go:278] ["starting DataCoord client for DataNode"] 2024-02-29 17:06:50.556 [2024/02/29 09:06:50.556 +00:00] [DEBUG] [sessionutil/session_util.go:254] ["Session try to connect to etcd"] 2024-02-29 17:06:50.556 [2024/02/29 09:06:50.556 +00:00] [DEBUG] [sessionutil/session_util.go:269] ["Session connect to etcd success"] 2024-02-29 17:06:50.557 [2024/02/29 09:06:50.557 +00:00] [DEBUG] [sessionutil/session_util.go:617] ["SessionUtil GetSessions"] [prefix=datacoord] [key=datacoord] [address=10.104.20.29:13333] 2024-02-29 17:06:50.557 [2024/02/29 09:06:50.557 +00:00] [DEBUG] [client/client.go:90] ["DataCoordClient GetSessions success"] [address=10.104.20.29:13333] [serverID=1] 2024-02-29 17:06:50.558 [2024/02/29 09:06:50.558 +00:00] [INFO] [componentutil/componentutil.go:61] ["WaitForComponentStates success"] ["current state"=Healthy] 2024-02-29 17:06:50.558 [2024/02/29 09:06:50.558 +00:00] [INFO] [datanode/service.go:289] ["DataCoord client is ready for DataNode"] 2024-02-29 17:06:50.558 [2024/02/29 09:06:50.558 +00:00] [INFO] [sync/once.go:74] ["DataNode server initializing"] [TimeTickChannelName=by-dev-datacoord-timetick-channel] 2024-02-29 17:06:50.558 [2024/02/29 09:06:50.558 +00:00] [DEBUG] [sessionutil/session_util.go:254] ["Session try to connect to etcd"] 2024-02-29 17:06:50.559 [2024/02/29 09:06:50.559 +00:00] [DEBUG] [sessionutil/session_util.go:269] ["Session connect to etcd success"] 2024-02-29 17:06:50.561 [2024/02/29 09:06:50.560 +00:00] [DEBUG] [sessionutil/session_util.go:312] [getServerID] [reuse=true] 2024-02-29 17:06:50.562 [2024/02/29 09:06:50.562 +00:00] [DEBUG] [sessionutil/session_util.go:370] ["Session get serverID success"] [key=id] [ServerId=9] 2024-02-29 17:06:50.562 [2024/02/29 09:06:50.562 +00:00] [INFO] [sessionutil/session_util.go:286] ["start server"] [name=datanode] [address=10.104.26.144:21124] [id=9] 2024-02-29 17:06:50.563 [2024/02/29 09:06:50.562 +00:00] [INFO] [sessionutil/session_util.go:1232] ["save server info into file"] [content="datanode-9\n"] [filePath=/tmp/milvus/server_id_8] 2024-02-29 17:06:50.563 [2024/02/29 09:06:50.563 +00:00] [INFO] [datanode/data_node.go:262] ["DataNode server init rateCollector done"] ["node ID"=0] 2024-02-29 17:06:50.563 [2024/02/29 09:06:50.563 +00:00] [INFO] [datanode/data_node.go:265] ["DataNode server init dispatcher client done"] ["node ID"=0] 2024-02-29 17:06:50.563 [2024/02/29 09:06:50.563 +00:00] [INFO] [dependency/factory.go:86] ["try to init mq"] [standalone=false] [mqType=pulsar] 2024-02-29 17:06:50.563 [2024/02/29 09:06:50.563 +00:00] [INFO] [datanode/data_node.go:278] ["DataNode server init succeeded"] [MsgChannelSubName=by-dev-dataNode] 2024-02-29 17:06:50.567 [2024/02/29 09:06:50.567 +00:00] [INFO] [storage/remote_chunk_manager.go:84] ["remote chunk manager init success."] [remote=aws] [bucketname=milvus-bucket] [root=file] 2024-02-29 17:06:50.567 [2024/02/29 09:06:50.567 +00:00] [INFO] [syncmgr/sync_manager.go:73] ["sync manager initialized"] [initPoolSize=256] 2024-02-29 17:06:50.567 [2024/02/29 09:06:50.567 +00:00] [INFO] [datanode/data_node.go:302] ["init datanode done"] [nodeID=0] [Address=10.104.26.144:21124] 2024-02-29 17:06:50.567 [2024/02/29 09:06:50.567 +00:00] [INFO] [datanode/service.go:301] ["current DataNode state"] [state=Initializing] 2024-02-29 17:06:50.567 [2024/02/29 09:06:50.567 +00:00] [INFO] [datanode/service.go:192] ["DataNode gRPC services successfully initialized"] 2024-02-29 17:06:50.567 [2024/02/29 09:06:50.567 +00:00] [INFO] [datanode/data_node.go:357] ["start id allocator done"] [role=datanode] 2024-02-29 17:06:50.567 [2024/02/29 09:06:50.567 +00:00] [DEBUG] [datanode/data_node.go:194] ["node begin to register to etcd"] [serverName=datanode] [ServerID=9] 2024-02-29 17:06:50.567 [2024/02/29 09:06:50.567 +00:00] [DEBUG] [sessionutil/session_util.go:433] ["service begin to register to etcd"] [serverName=datanode] [ServerID=9] 2024-02-29 17:06:50.567 [2024/02/29 09:06:50.567 +00:00] [INFO] [importv2/executor.go:73] ["start import executor"] 2024-02-29 17:06:50.567 [2024/02/29 09:06:50.567 +00:00] [INFO] [datanode/data_node.go:336] ["DataNode Background GC Start"] 2024-02-29 17:06:50.567 [2024/02/29 09:06:50.567 +00:00] [INFO] [datanode/event_manager.go:52] ["Start watch channel"] [prefix=channelwatch/9] 2024-02-29 17:06:50.568 [2024/02/29 09:06:50.568 +00:00] [INFO] [sessionutil/session_util.go:463] ["put session key into etcd"] [key=by-dev/meta/session/datanode-9] [value="{\"ServerID\":9,\"ServerName\":\"datanode\",\"Address\":\"10.104.26.144:21124\",\"TriggerKill\":true,\"Version\":\"2.3.2\",\"IndexEngineVersion\":{},\"LeaseID\":741279687924912288,\"HostName\":\"inverted-corn-c79tm-5-49-7343-milvus-datanode-54f56c4cfb-4bwbv\"}"] 2024-02-29 17:06:50.568 [2024/02/29 09:06:50.568 +00:00] [INFO] [sessionutil/session_util.go:473] ["Service registered successfully"] [ServerName=datanode] [serverID=9] 2024-02-29 17:06:50.568 [2024/02/29 09:06:50.568 +00:00] [INFO] [datanode/data_node.go:198] ["DataNode Register Finished"] 2024-02-29 17:06:50.569 [2024/02/29 09:06:50.569 +00:00] [INFO] [datanode/service.go:197] ["DataNode gRPC services successfully started"] 2024-02-29 17:06:50.569 [2024/02/29 09:06:50.569 +00:00] [DEBUG] [components/data_node.go:57] ["Datanode successfully started"] 2024-02-29 17:06:50.569 [2024/02/29 09:06:50.569 +00:00] [INFO] [logutil/logutil.go:163] ["Log directory"] [configDir=] 2024-02-29 17:06:50.569 [2024/02/29 09:06:50.569 +00:00] [INFO] [logutil/logutil.go:164] ["Set log file to "] [path=] 2024-02-29 17:06:50.569 [2024/02/29 09:06:50.569 +00:00] [INFO] [tracer/tracer.go:78] ["Init tracer finished"] [Exporter=stdout] 2024-02-29 17:06:50.573 [2024/02/29 09:06:50.573 +00:00] [INFO] [datanode/event_manager.go:127] ["DataNode is handling watchInfo PUT event"] [key=by-dev/meta/channelwatch/9/by-dev-rootcoord-dml_0_448050465291633127v0] ["watch state"=ToWatch] 2024-02-29 17:06:50.573 [2024/02/29 09:06:50.573 +00:00] [INFO] [datanode/event_manager.go:127] ["DataNode is handling watchInfo PUT event"] [key=by-dev/meta/channelwatch/9/by-dev-rootcoord-dml_1_448050465291633127v1] ["watch state"=ToWatch] 2024-02-29 17:06:50.573 [2024/02/29 09:06:50.573 +00:00] [WARN] [datanode/flow_graph_manager.go:65] ["debug AddandStartWithEtcdTickler 0"] [channel=by-dev-rootcoord-dml_1_448050465291633127v1] 2024-02-29 17:06:50.573 [2024/02/29 09:06:50.573 +00:00] [WARN] [datanode/flow_graph_manager.go:65] ["debug AddandStartWithEtcdTickler 0"] [channel=by-dev-rootcoord-dml_0_448050465291633127v0] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.625 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_0_448050465291633127v0] [segmentID=448050465293336255] [numRows=5] [segmentType=sealed] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_0_448050465291633127v0] [segmentID=448050465293336122] [numRows=6] [segmentType=sealed] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_0_448050465291633127v0] [segmentID=448050465293336250] [numRows=12] [segmentType=sealed] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_0_448050465291633127v0] [segmentID=448050465293336119] [numRows=4] [segmentType=sealed] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_0_448050465291633127v0] [segmentID=448050465293336241] [numRows=10] [segmentType=sealed] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_0_448050465291633127v0] [segmentID=448050465293336347] [numRows=3] [segmentType=sealed] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_0_448050465291633127v0] [segmentID=448050465293336068] [numRows=6] [segmentType=sealed] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_0_448050465291633127v0] [segmentID=448050465293336078] [numRows=18] [segmentType=sealed] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_0_448050465291633127v0] [segmentID=448050465293336104] [numRows=9] [segmentType=sealed] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_0_448050465291633127v0] [segmentID=448050465293336151] [numRows=9] [segmentType=sealed] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_0_448050465291633127v0] [segmentID=448050465293336280] [numRows=3] [segmentType=sealed] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_0_448050465291633127v0] [segmentID=448050465293336248] [numRows=6] [segmentType=sealed] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_0_448050465291633127v0] [segmentID=448050465291860113] [numRows=149440] [segmentType=sealed] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_0_448050465291633127v0] [segmentID=448050465293336102] [numRows=2] [segmentType=sealed] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_0_448050465291633127v0] [segmentID=448050465293336252] [numRows=13] [segmentType=sealed] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336252] [statsBinLogsLen=1] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336255] [statsBinLogsLen=1] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336122] [statsBinLogsLen=1] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336250] [statsBinLogsLen=1] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336119] [statsBinLogsLen=1] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336241] [statsBinLogsLen=1] 2024-02-29 17:06:50.626 [2024/02/29 09:06:50.626 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336347] [statsBinLogsLen=1] 2024-02-29 17:06:50.627 [2024/02/29 09:06:50.627 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336068] [statsBinLogsLen=1] 2024-02-29 17:06:50.627 [2024/02/29 09:06:50.627 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336078] [statsBinLogsLen=1] 2024-02-29 17:06:50.627 [2024/02/29 09:06:50.627 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336104] [statsBinLogsLen=1] 2024-02-29 17:06:50.627 [2024/02/29 09:06:50.627 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336151] [statsBinLogsLen=1] 2024-02-29 17:06:50.627 [2024/02/29 09:06:50.627 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] [segmentID=448050465293336086] [numRows=5] [segmentType=sealed] 2024-02-29 17:06:50.627 [2024/02/29 09:06:50.627 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336280] [statsBinLogsLen=1] 2024-02-29 17:06:50.627 [2024/02/29 09:06:50.627 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] [segmentID=448050465293336246] [numRows=4] [segmentType=sealed] 2024-02-29 17:06:50.627 [2024/02/29 09:06:50.627 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336086] [statsBinLogsLen=1] 2024-02-29 17:06:50.627 [2024/02/29 09:06:50.627 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336248] [statsBinLogsLen=1] 2024-02-29 17:06:50.627 [2024/02/29 09:06:50.627 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465291860113] [statsBinLogsLen=1] 2024-02-29 17:06:50.627 [2024/02/29 09:06:50.627 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336102] [statsBinLogsLen=1] 2024-02-29 17:06:50.634 [2024/02/29 09:06:50.634 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336086] [time=7.550605ms] [size=32] 2024-02-29 17:06:50.634 [2024/02/29 09:06:50.634 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336252] [time=8.792984ms] [size=82] 2024-02-29 17:06:50.634 [2024/02/29 09:06:50.634 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] [segmentID=448050465293336348] [numRows=7] [segmentType=sealed] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.634 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] [segmentID=448050465293336069] [numRows=4] [segmentType=sealed] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.634 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336348] [statsBinLogsLen=1] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.634 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336104] [time=7.831254ms] [size=57] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] [segmentID=448050465293336079] [numRows=7] [segmentType=sealed] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336069] [statsBinLogsLen=1] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336246] [statsBinLogsLen=1] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336241] [time=8.303805ms] [size=63] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] [segmentID=448050465293336249] [numRows=18] [segmentType=sealed] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336079] [statsBinLogsLen=1] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336280] [time=8.009168ms] [size=19] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] [segmentID=448050465291858599] [numRows=150560] [segmentType=sealed] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336078] [time=8.225054ms] [size=113] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336249] [statsBinLogsLen=1] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] [segmentID=448050465293336281] [numRows=7] [segmentType=sealed] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465291858599] [statsBinLogsLen=1] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336255] [time=9.177326ms] [size=32] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] [segmentID=448050465293336120] [numRows=6] [segmentType=sealed] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336281] [statsBinLogsLen=1] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336347] [time=8.828726ms] [size=19] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336248] [time=8.427667ms] [size=38] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] [segmentID=448050465293336256] [numRows=5] [segmentType=sealed] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] [segmentID=448050465293336103] [numRows=16] [segmentType=sealed] 2024-02-29 17:06:50.635 [2024/02/29 09:06:50.635 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336256] [statsBinLogsLen=1] 2024-02-29 17:06:50.636 [2024/02/29 09:06:50.636 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336120] [statsBinLogsLen=1] 2024-02-29 17:06:50.636 [2024/02/29 09:06:50.636 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336151] [time=8.969103ms] [size=57] 2024-02-29 17:06:50.636 [2024/02/29 09:06:50.636 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] [segmentID=448050465293336239] [numRows=10] [segmentType=sealed] 2024-02-29 17:06:50.636 [2024/02/29 09:06:50.636 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336103] [statsBinLogsLen=1] 2024-02-29 17:06:50.636 [2024/02/29 09:06:50.636 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336068] [time=9.389609ms] [size=38] 2024-02-29 17:06:50.636 [2024/02/29 09:06:50.636 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] [segmentID=448050465293336105] [numRows=3] [segmentType=sealed] 2024-02-29 17:06:50.636 [2024/02/29 09:06:50.636 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336239] [statsBinLogsLen=1] 2024-02-29 17:06:50.636 [2024/02/29 09:06:50.636 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336122] [time=9.9746ms] [size=38] 2024-02-29 17:06:50.636 [2024/02/29 09:06:50.636 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] [segmentID=448050465293336253] [numRows=7] [segmentType=sealed] 2024-02-29 17:06:50.636 [2024/02/29 09:06:50.636 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336105] [statsBinLogsLen=1] 2024-02-29 17:06:50.636 [2024/02/29 09:06:50.636 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336119] [time=10.016777ms] [size=25] 2024-02-29 17:06:50.636 [2024/02/29 09:06:50.636 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] [segmentID=448050465293336152] [numRows=11] [segmentType=sealed] 2024-02-29 17:06:50.636 [2024/02/29 09:06:50.636 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336253] [statsBinLogsLen=1] 2024-02-29 17:06:50.637 [2024/02/29 09:06:50.637 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336102] [time=9.572822ms] [size=13] 2024-02-29 17:06:50.637 [2024/02/29 09:06:50.637 +00:00] [INFO] [datanode/data_sync_service.go:146] ["recover segments from checkpoints"] [vChannelName=by-dev-rootcoord-dml_1_448050465291633127v1] [segmentID=448050465293336121] [numRows=4] [segmentType=sealed] 2024-02-29 17:06:50.637 [2024/02/29 09:06:50.637 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336152] [statsBinLogsLen=1] 2024-02-29 17:06:50.637 [2024/02/29 09:06:50.637 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336250] [time=10.547464ms] [size=75] 2024-02-29 17:06:50.637 [2024/02/29 09:06:50.637 +00:00] [INFO] [datanode/data_sync_service.go:249] ["begin to init pk bloom filter"] [segmentID=448050465293336121] [statsBinLogsLen=1] 2024-02-29 17:06:50.639 [2024/02/29 09:06:50.639 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465291860113] [time=12.498711ms] [size=931792] 2024-02-29 17:06:50.640 [2024/02/29 09:06:50.640 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336069] [time=5.40977ms] [size=25] 2024-02-29 17:06:50.640 [2024/02/29 09:06:50.640 +00:00] [INFO] [msgdispatcher/manager.go:67] ["create new dispatcherManager"] [role=datanode] [nodeID=0] [pchannel=by-dev-rootcoord-dml_0] 2024-02-29 17:06:50.640 [2024/02/29 09:06:50.640 +00:00] [INFO] [msgdispatcher/dispatcher.go:93] ["creating dispatcher..."] [pchannel=by-dev-rootcoord-dml_0] [subName=datanode-0-by-dev-rootcoord-dml_0_448050465291633127v0-true] [isMain=true] 2024-02-29 17:06:50.641 [2024/02/29 09:06:50.641 +00:00] [INFO] [msgstream/mq_msgstream.go:116] ["Msg Stream state"] [can_produce=true] 2024-02-29 17:06:50.641 [2024/02/29 09:06:50.641 +00:00] [INFO] [msgdispatcher/manager.go:155] ["dispatcherManager is running..."] [role=datanode] [nodeID=0] [pchannel=by-dev-rootcoord-dml_0] 2024-02-29 17:06:50.642 [2024/02/29 09:06:50.642 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336121] [time=4.851297ms] [size=25] 2024-02-29 17:06:50.642 [2024/02/29 09:06:50.642 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336079] [time=7.045198ms] [size=44] 2024-02-29 17:06:50.642 [2024/02/29 09:06:50.642 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336256] [time=6.478593ms] [size=32] 2024-02-29 17:06:50.642 [2024/02/29 09:06:50.642 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336348] [time=7.641977ms] [size=44] 2024-02-29 17:06:50.642 [2024/02/29 09:06:50.642 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336105] [time=6.186326ms] [size=19] 2024-02-29 17:06:50.643 [2024/02/29 09:06:50.643 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336120] [time=7.43954ms] [size=38] 2024-02-29 17:06:50.643 [2024/02/29 09:06:50.643 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336246] [time=8.354549ms] [size=25] 2024-02-29 17:06:50.643 [2024/02/29 09:06:50.643 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336103] [time=7.502561ms] [size=100] 2024-02-29 17:06:50.643 [2024/02/29 09:06:50.643 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336281] [time=8.202696ms] [size=44] 2024-02-29 17:06:50.643 [2024/02/29 09:06:50.643 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336249] [time=8.557008ms] [size=113] 2024-02-29 17:06:50.644 [2024/02/29 09:06:50.644 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336253] [time=7.472827ms] [size=44] 2024-02-29 17:06:50.644 [2024/02/29 09:06:50.644 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336152] [time=7.243416ms] [size=69] 2024-02-29 17:06:50.644 [2024/02/29 09:06:50.644 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465293336239] [time=8.080561ms] [size=63] 2024-02-29 17:06:50.647 [2024/02/29 09:06:50.647 +00:00] [INFO] [datanode/data_sync_service.go:328] ["Successfully load pk stats"] [segmentID=448050465291858599] [time=11.673814ms] [size=938776] 2024-02-29 17:06:50.654 [2024/02/29 09:06:50.654 +00:00] [INFO] [sessionutil/session_util.go:910] ["register session success"] [role=datanode] [key=by-dev/meta/session/datanode-9] 2024-02-29 17:06:50.678 [2024/02/29 09:06:50.677 +00:00] [INFO] [msgstream/mq_msgstream.go:853] ["MsgStream begin to seek start msg: "] [channel=by-dev-rootcoord-dml_0] [MessageID=CCEQqxgYACAA]