Common labels: {"app":"milvus","cluster":"4am","component":"querycoord","container":"querycoord","job":"qa-milvus/milvus","namespace":"qa-milvus","node_name":"4am-node18","pod":"inverted-corn-136000-8-75-4542-milvus-querycoord-6c7f585db2t9qm","stream":"stdout"} Line limit: "5000 reached, received logs cover 64.52% (26sec) of your selected time range (40sec)" Total bytes processed: "2.16 MB" 2024-02-29 01:31:03.192 [2024/02/28 17:31:03.190 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626357] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:03.192 [2024/02/28 17:31:03.190 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626321] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760422] 2024-02-29 01:31:03.192 [2024/02/28 17:31:03.191 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626333] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:03.192 [2024/02/28 17:31:03.191 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:03.192 [2024/02/28 17:31:03.191 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626333] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759968] 2024-02-29 01:31:03.192 [2024/02/28 17:31:03.191 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626330] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:03.192 [2024/02/28 17:31:03.191 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:03.192 [2024/02/28 17:31:03.191 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626330] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760199] 2024-02-29 01:31:03.192 [2024/02/28 17:31:03.192 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626307] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:03.192 [2024/02/28 17:31:03.192 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:03.192 [2024/02/28 17:31:03.192 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626307] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760430] 2024-02-29 01:31:03.192 [2024/02/28 17:31:03.192 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626318] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:03.192 [2024/02/28 17:31:03.192 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:03.192 [2024/02/28 17:31:03.192 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626318] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760748] 2024-02-29 01:31:03.192 [2024/02/28 17:31:03.192 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=38] [committedNum=27] [toRemoveNum=5] 2024-02-29 01:31:03.193 [2024/02/28 17:31:03.192 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=38] [waitingTaskNum=0] [segmentTaskNum=38] [channelTaskNum=0] 2024-02-29 01:31:03.200 [2024/02/28 17:31:03.200 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626350] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:03.201 [2024/02/28 17:31:03.201 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626352] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:03.202 [2024/02/28 17:31:03.202 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626348] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:03.204 [2024/02/28 17:31:03.204 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626350] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] [elapsed=3.505414ms] 2024-02-29 01:31:03.206 [2024/02/28 17:31:03.206 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626348] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] [elapsed=3.980791ms] 2024-02-29 01:31:03.236 [2024/02/28 17:31:03.236 +00:00] [WARN] [task/executor.go:468] ["failed to sync distribution"] [taskID=1709136626352] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:03.236 [2024/02/28 17:31:03.236 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626352] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:03.664 [2024/02/28 17:31:03.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626381] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:03.664 [2024/02/28 17:31:03.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626382] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:03.664 [2024/02/28 17:31:03.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626383] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:03.664 [2024/02/28 17:31:03.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626384] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:03.664 [2024/02/28 17:31:03.664 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:03.664 [2024/02/28 17:31:03.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:03.664 [2024/02/28 17:31:03.664 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:03.664 [2024/02/28 17:31:03.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626525234] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626527784] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626553190] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626547521] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.665 [2024/02/28 17:31:03.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:03.666 [2024/02/28 17:31:03.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626385] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760430]"] 2024-02-29 01:31:03.666 [2024/02/28 17:31:03.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626386] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760748]"] 2024-02-29 01:31:03.666 [2024/02/28 17:31:03.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626387] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760422]"] 2024-02-29 01:31:03.666 [2024/02/28 17:31:03.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626388] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760199]"] 2024-02-29 01:31:03.666 [2024/02/28 17:31:03.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626389] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759968]"] 2024-02-29 01:31:03.667 [2024/02/28 17:31:03.667 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626383] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.667 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626383] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.668 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626383] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.668 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626382] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.668 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626382] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.668 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626382] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.668 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626381] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.668 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626381] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.668 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626381] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.668 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626384] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.668 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626384] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.668 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626384] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.668 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626383] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.668 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626382] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.668 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626381] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.668 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626384] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.668 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=5] [toRemoveNum=4] 2024-02-29 01:31:03.668 [2024/02/28 17:31:03.668 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=43] [waitingTaskNum=0] [segmentTaskNum=43] [channelTaskNum=0] 2024-02-29 01:31:03.669 [2024/02/28 17:31:03.669 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626387] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:03.669 [2024/02/28 17:31:03.669 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626389] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:03.669 [2024/02/28 17:31:03.669 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626385] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:03.669 [2024/02/28 17:31:03.669 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626386] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:03.669 [2024/02/28 17:31:03.669 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626388] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:03.669 [2024/02/28 17:31:03.669 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=40] ["time cost"=274.858µs] 2024-02-29 01:31:03.669 [2024/02/28 17:31:03.669 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626350] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626528386] 2024-02-29 01:31:03.669 [2024/02/28 17:31:03.669 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626352] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626524214] 2024-02-29 01:31:03.669 [2024/02/28 17:31:03.669 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626348] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626527785] 2024-02-29 01:31:03.669 [2024/02/28 17:31:03.669 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=40] [committedNum=5] [toRemoveNum=3] 2024-02-29 01:31:03.669 [2024/02/28 17:31:03.669 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=40] [waitingTaskNum=0] [segmentTaskNum=40] [channelTaskNum=0] 2024-02-29 01:31:03.870 [2024/02/28 17:31:03.870 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136625265] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626548132] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626548132/448039877626558600]"] 2024-02-29 01:31:03.870 [2024/02/28 17:31:03.870 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136625265] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626548132/448039877626558600]"] 2024-02-29 01:31:03.911 [2024/02/28 17:31:03.911 +00:00] [DEBUG] [config/etcd_source.go:143] ["etcd refreshConfigurations"] [prefix=by-dev/config] [endpoints="[inverted-corn-136000-8-75-4542-etcd:2379]"] 2024-02-29 01:31:04.186 [2024/02/28 17:31:04.186 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=40] [waitingTaskNum=0] [segmentTaskNum=40] [channelTaskNum=0] 2024-02-29 01:31:04.190 [2024/02/28 17:31:04.189 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=39] ["time cost"=1.716798ms] 2024-02-29 01:31:04.190 [2024/02/28 17:31:04.189 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136625265] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626548132] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626548132/448039877626558600]"] 2024-02-29 01:31:04.190 [2024/02/28 17:31:04.190 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626548132/448039877626558600]"] 2024-02-29 01:31:04.190 [2024/02/28 17:31:04.190 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136625265] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626548132] 2024-02-29 01:31:04.190 [2024/02/28 17:31:04.190 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=39] [committedNum=0] [toRemoveNum=1] 2024-02-29 01:31:04.190 [2024/02/28 17:31:04.190 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=39] [waitingTaskNum=0] [segmentTaskNum=39] [channelTaskNum=0] 2024-02-29 01:31:04.663 [2024/02/28 17:31:04.663 +00:00] [DEBUG] [checkers/leader_checker.go:138] ["leader checker append a segment to set"] [collectionID=448039877626298924] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [leaderViewID=3] [segmentID=448039877626555509] [nodeID=3] 2024-02-29 01:31:04.663 [2024/02/28 17:31:04.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626390] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:04.663 [2024/02/28 17:31:04.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626391] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:04.663 [2024/02/28 17:31:04.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626392] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626528386][leader=3]"] 2024-02-29 01:31:04.663 [2024/02/28 17:31:04.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626393] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:04.663 [2024/02/28 17:31:04.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626394] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:04.663 [2024/02/28 17:31:04.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626395] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626527785][leader=3]"] 2024-02-29 01:31:04.663 [2024/02/28 17:31:04.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626396] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626524214][leader=3]"] 2024-02-29 01:31:04.665 [2024/02/28 17:31:04.665 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626553190] [error="node offline[node=8]"] 2024-02-29 01:31:04.665 [2024/02/28 17:31:04.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:04.665 [2024/02/28 17:31:04.665 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:04.665 [2024/02/28 17:31:04.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:04.665 [2024/02/28 17:31:04.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.665 [2024/02/28 17:31:04.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.665 [2024/02/28 17:31:04.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626547521] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.665 [2024/02/28 17:31:04.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.665 [2024/02/28 17:31:04.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.665 [2024/02/28 17:31:04.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.665 [2024/02/28 17:31:04.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.665 [2024/02/28 17:31:04.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.665 [2024/02/28 17:31:04.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.665 [2024/02/28 17:31:04.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626525234] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626527784] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626553190] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:04.666 [2024/02/28 17:31:04.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:04.667 [2024/02/28 17:31:04.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626397] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877626548132]"] 2024-02-29 01:31:04.674 [2024/02/28 17:31:04.674 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626391] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:04.674 [2024/02/28 17:31:04.674 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626391] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:04.674 [2024/02/28 17:31:04.674 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626391] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:04.674 [2024/02/28 17:31:04.674 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626394] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:04.674 [2024/02/28 17:31:04.674 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626394] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:04.674 [2024/02/28 17:31:04.674 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626394] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:04.674 [2024/02/28 17:31:04.674 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626390] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:04.675 [2024/02/28 17:31:04.674 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626390] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:04.675 [2024/02/28 17:31:04.675 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626390] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:04.675 [2024/02/28 17:31:04.675 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626393] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:04.675 [2024/02/28 17:31:04.675 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626393] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:04.675 [2024/02/28 17:31:04.675 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626393] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:04.675 [2024/02/28 17:31:04.675 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626391] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:04.675 [2024/02/28 17:31:04.675 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626394] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:04.675 [2024/02/28 17:31:04.675 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626390] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:04.675 [2024/02/28 17:31:04.675 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626393] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:04.675 [2024/02/28 17:31:04.675 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=4] [toRemoveNum=4] 2024-02-29 01:31:04.675 [2024/02/28 17:31:04.675 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=43] [waitingTaskNum=0] [segmentTaskNum=43] [channelTaskNum=0] 2024-02-29 01:31:04.677 [2024/02/28 17:31:04.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626392] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:04.678 [2024/02/28 17:31:04.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626396] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:04.678 [2024/02/28 17:31:04.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626397] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:04.678 [2024/02/28 17:31:04.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626395] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:04.678 [2024/02/28 17:31:04.678 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=43] ["time cost"=1.019539ms] 2024-02-29 01:31:04.678 [2024/02/28 17:31:04.678 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=43] [committedNum=4] [toRemoveNum=0] 2024-02-29 01:31:04.678 [2024/02/28 17:31:04.678 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=43] [waitingTaskNum=0] [segmentTaskNum=43] [channelTaskNum=0] 2024-02-29 01:31:04.888 [2024/02/28 17:31:04.888 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626380] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.067 [2024/02/28 17:31:05.067 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626372] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.067 [2024/02/28 17:31:05.067 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626357] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.083 [2024/02/28 17:31:05.082 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626373] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.094 [2024/02/28 17:31:05.084 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626377] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.094 [2024/02/28 17:31:05.085 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626369] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.094 [2024/02/28 17:31:05.085 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626364] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.094 [2024/02/28 17:31:05.086 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626367] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.094 [2024/02/28 17:31:05.086 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626378] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.094 [2024/02/28 17:31:05.087 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626356] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.103 [2024/02/28 17:31:05.103 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626366] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.103 [2024/02/28 17:31:05.103 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626362] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.110 [2024/02/28 17:31:05.110 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626375] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.130 [2024/02/28 17:31:05.130 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626386] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.130 [2024/02/28 17:31:05.130 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626379] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.130 [2024/02/28 17:31:05.130 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626358] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.131 [2024/02/28 17:31:05.131 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626376] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.163 [2024/02/28 17:31:05.163 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626370] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.163 [2024/02/28 17:31:05.163 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626365] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.164 [2024/02/28 17:31:05.163 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626354] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.164 [2024/02/28 17:31:05.163 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626361] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.164 [2024/02/28 17:31:05.163 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626371] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.168 [2024/02/28 17:31:05.168 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=43] [waitingTaskNum=0] [segmentTaskNum=43] [channelTaskNum=0] 2024-02-29 01:31:05.169 [2024/02/28 17:31:05.169 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=43] ["time cost"=269.645µs] 2024-02-29 01:31:05.169 [2024/02/28 17:31:05.169 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=43] [committedNum=0] [toRemoveNum=0] 2024-02-29 01:31:05.169 [2024/02/28 17:31:05.169 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=43] [waitingTaskNum=0] [segmentTaskNum=43] [channelTaskNum=0] 2024-02-29 01:31:05.200 [2024/02/28 17:31:05.200 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626374] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.230 [2024/02/28 17:31:05.230 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626363] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.249 [2024/02/28 17:31:05.248 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626389] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.266 [2024/02/28 17:31:05.266 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626359] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.266 [2024/02/28 17:31:05.266 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626368] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.283 [2024/02/28 17:31:05.282 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626388] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.283 [2024/02/28 17:31:05.283 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626360] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.287 [2024/02/28 17:31:05.287 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626355] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.305 [2024/02/28 17:31:05.304 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:05.305 [2024/02/28 17:31:05.305 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=029dd5992e0cebbf3d676ead80b268b4] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:05.315 [2024/02/28 17:31:05.315 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626387] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.360 [2024/02/28 17:31:05.360 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:05.360 [2024/02/28 17:31:05.360 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=e3a9b378e6e3318108e6838602103a3c] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:05.509 [2024/02/28 17:31:05.509 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:05.509 [2024/02/28 17:31:05.509 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=4426af8d10e10d7fd6c20806bd26faae] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:05.525 [2024/02/28 17:31:05.524 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626385] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.537 [2024/02/28 17:31:05.537 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626396] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:05.553 [2024/02/28 17:31:05.553 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626397] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626548132] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:05.585 [2024/02/28 17:31:05.584 +00:00] [WARN] [task/executor.go:468] ["failed to sync distribution"] [taskID=1709136626396] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:05.585 [2024/02/28 17:31:05.584 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626396] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:05.597 [2024/02/28 17:31:05.597 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626395] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:05.603 [2024/02/28 17:31:05.603 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626395] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] [elapsed=6.118949ms] 2024-02-29 01:31:05.604 [2024/02/28 17:31:05.604 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626392] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:05.622 [2024/02/28 17:31:05.622 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626392] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] [elapsed=17.593271ms] 2024-02-29 01:31:05.663 [2024/02/28 17:31:05.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626398] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:05.663 [2024/02/28 17:31:05.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626399] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:05.663 [2024/02/28 17:31:05.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626400] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:05.663 [2024/02/28 17:31:05.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626401] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:05.665 [2024/02/28 17:31:05.665 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626527784] [error="node offline[node=8]"] 2024-02-29 01:31:05.665 [2024/02/28 17:31:05.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:05.665 [2024/02/28 17:31:05.665 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:05.665 [2024/02/28 17:31:05.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626547521] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626525234] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626527784] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626553190] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:05.666 [2024/02/28 17:31:05.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:05.670 [2024/02/28 17:31:05.670 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626400] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:05.670 [2024/02/28 17:31:05.670 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626400] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:05.670 [2024/02/28 17:31:05.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626400] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:05.670 [2024/02/28 17:31:05.670 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626399] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:05.670 [2024/02/28 17:31:05.670 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626399] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:05.670 [2024/02/28 17:31:05.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626399] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:05.670 [2024/02/28 17:31:05.670 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626398] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:05.670 [2024/02/28 17:31:05.670 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626398] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:05.670 [2024/02/28 17:31:05.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626398] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:05.670 [2024/02/28 17:31:05.670 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626401] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:05.670 [2024/02/28 17:31:05.670 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626401] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:05.671 [2024/02/28 17:31:05.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626401] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:05.672 [2024/02/28 17:31:05.670 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626400] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:05.672 [2024/02/28 17:31:05.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626399] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:05.672 [2024/02/28 17:31:05.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626398] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:05.672 [2024/02/28 17:31:05.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626401] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:05.672 [2024/02/28 17:31:05.671 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=0] [toRemoveNum=4] 2024-02-29 01:31:05.672 [2024/02/28 17:31:05.671 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=43] [waitingTaskNum=0] [segmentTaskNum=43] [channelTaskNum=0] 2024-02-29 01:31:05.696 [2024/02/28 17:31:05.696 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=40] ["time cost"=23.547691ms] 2024-02-29 01:31:05.697 [2024/02/28 17:31:05.696 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626396] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626524214] 2024-02-29 01:31:05.697 [2024/02/28 17:31:05.696 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626395] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626527785] 2024-02-29 01:31:05.697 [2024/02/28 17:31:05.697 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626392] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626528386] 2024-02-29 01:31:05.697 [2024/02/28 17:31:05.697 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=40] [committedNum=0] [toRemoveNum=3] 2024-02-29 01:31:05.697 [2024/02/28 17:31:05.697 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=40] [waitingTaskNum=0] [segmentTaskNum=40] [channelTaskNum=0] 2024-02-29 01:31:05.703 [2024/02/28 17:31:05.703 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:05.703 [2024/02/28 17:31:05.703 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=54e5aa6ddf4229ac42853fc7760b6a84] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:05.756 [2024/02/28 17:31:05.756 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:05.756 [2024/02/28 17:31:05.756 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=fd1f528b26c713f750cabd4fefda0738] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:05.825 [2024/02/28 17:31:05.824 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=8db53fb52d0997c14734a3d02e2dba38] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626299033,448039877626298993,448039877626298987,448039877626298965,448039877626299005,448039877626299022,448039877626298959,448039877626299016,448039877626298970,448039877626298981,448039877626299027,448039877626298936,448039877626298931,448039877626299011,448039877626298953,448039877626298976,448039877626298942,448039877626298948,448039877626298999]"] 2024-02-29 01:31:05.825 [2024/02/28 17:31:05.824 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=8db53fb52d0997c14734a3d02e2dba38] [collectionID=448039877626298924] 2024-02-29 01:31:05.825 [2024/02/28 17:31:05.824 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=8db53fb52d0997c14734a3d02e2dba38] [collectionID=448039877626298924] 2024-02-29 01:31:05.825 [2024/02/28 17:31:05.825 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:05.826 [2024/02/28 17:31:05.825 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=8db53fb52d0997c14734a3d02e2dba38] [collectionID=448039877626298924] 2024-02-29 01:31:05.826 [2024/02/28 17:31:05.826 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=8db53fb52d0997c14734a3d02e2dba38] [collectionID=448039877626298924] 2024-02-29 01:31:05.831 [2024/02/28 17:31:05.831 +00:00] [DEBUG] [querycoordv2/server.go:528] ["QueryCoord current state"] [StateCode=Healthy] 2024-02-29 01:31:05.865 [2024/02/28 17:31:05.865 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:05.865 [2024/02/28 17:31:05.865 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=83a400385f8e157e65f88739b216ffa4] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:05.867 [2024/02/28 17:31:05.867 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:05.867 [2024/02/28 17:31:05.867 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=52da378b249e0987ce8082182576c2ef] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:05.869 [2024/02/28 17:31:05.869 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:05.869 [2024/02/28 17:31:05.869 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=f7d89699575e842146cdc2cee0bcce94] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:05.882 [2024/02/28 17:31:05.874 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:05.882 [2024/02/28 17:31:05.875 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=4d7fdbe3aa02dfc876f0d3fb22f38a37] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:05.990 [2024/02/28 17:31:05.989 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:05.990 [2024/02/28 17:31:05.990 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=8dbfc9be0e951610a81a987fed22e45c] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:05.990 [2024/02/28 17:31:05.990 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:05.990 [2024/02/28 17:31:05.990 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=84459d9e3a2d6250cbe35a906d085067] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:05.995 [2024/02/28 17:31:05.995 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:05.995 [2024/02/28 17:31:05.995 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=243871c31d6c8d1ad640c135abf11a2d] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:05.996 [2024/02/28 17:31:05.995 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:05.996 [2024/02/28 17:31:05.995 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=242360376eb7097b89f5f09739afb973] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:06.095 [2024/02/28 17:31:06.095 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:06.095 [2024/02/28 17:31:06.095 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=157506b9732be92ac73edf06be6659c6] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:06.096 [2024/02/28 17:31:06.096 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=93770717cc16337a6cfb74079e51843a] [collections="[448039877626298924]"] 2024-02-29 01:31:06.169 [2024/02/28 17:31:06.169 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=40] [waitingTaskNum=0] [segmentTaskNum=40] [channelTaskNum=0] 2024-02-29 01:31:06.170 [2024/02/28 17:31:06.170 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=40] ["time cost"=163.874µs] 2024-02-29 01:31:06.170 [2024/02/28 17:31:06.170 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=40] [committedNum=0] [toRemoveNum=0] 2024-02-29 01:31:06.170 [2024/02/28 17:31:06.170 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=40] [waitingTaskNum=0] [segmentTaskNum=40] [channelTaskNum=0] 2024-02-29 01:31:06.196 [2024/02/28 17:31:06.196 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:06.196 [2024/02/28 17:31:06.196 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=831c3539b367f322de407c283a51f9dc] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:06.256 [2024/02/28 17:31:06.256 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626527182] 2024-02-29 01:31:06.256 [2024/02/28 17:31:06.256 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=8d70640e40e9e6bdf475dacb1312052a] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626527182]"] 2024-02-29 01:31:06.307 [2024/02/28 17:31:06.307 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:06.307 [2024/02/28 17:31:06.307 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=73225efd80c60fce78058cb0482d3d68] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:06.409 [2024/02/28 17:31:06.409 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:06.409 [2024/02/28 17:31:06.409 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=8561558d3ffaaaa44f86a5c2e49f0517] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:06.512 [2024/02/28 17:31:06.511 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:06.512 [2024/02/28 17:31:06.512 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=f1e164e9b379aa4ecb7e354b9c66fd3a] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:06.514 [2024/02/28 17:31:06.514 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626525234] [error="node offline[node=8]"] 2024-02-29 01:31:06.514 [2024/02/28 17:31:06.514 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_1_448039877626298924v1 is not available in any replica"] [traceID=3509ef29a268d0222a9c3d76c2c16285] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:06.515 [2024/02/28 17:31:06.515 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626525234] [error="node offline[node=8]"] 2024-02-29 01:31:06.515 [2024/02/28 17:31:06.515 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_1_448039877626298924v1 is not available in any replica"] [traceID=c2299ce8e88e24ec2d0e944621701310] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:06.518 [2024/02/28 17:31:06.517 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:06.518 [2024/02/28 17:31:06.518 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=2d227f25ce47fe38b5937fab75e0f1d7] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:06.559 [2024/02/28 17:31:06.559 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=824a62d7948220b9a0d8f4b428f58546] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298942,448039877626298987,448039877626298953,448039877626298948,448039877626298965,448039877626299033,448039877626299027,448039877626298931,448039877626298999,448039877626298959,448039877626298993,448039877626299005,448039877626298976,448039877626298970,448039877626299022,448039877626299011,448039877626299016,448039877626298981,448039877626298936]"] 2024-02-29 01:31:06.559 [2024/02/28 17:31:06.559 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=824a62d7948220b9a0d8f4b428f58546] [collectionID=448039877626298924] 2024-02-29 01:31:06.559 [2024/02/28 17:31:06.559 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=824a62d7948220b9a0d8f4b428f58546] [collectionID=448039877626298924] 2024-02-29 01:31:06.559 [2024/02/28 17:31:06.559 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:06.560 [2024/02/28 17:31:06.560 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=824a62d7948220b9a0d8f4b428f58546] [collectionID=448039877626298924] 2024-02-29 01:31:06.560 [2024/02/28 17:31:06.560 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=824a62d7948220b9a0d8f4b428f58546] [collectionID=448039877626298924] 2024-02-29 01:31:06.599 [2024/02/28 17:31:06.598 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=ece0b73aea8a045c222364f8a6b3eec3] [collections="[448039877626298924]"] 2024-02-29 01:31:06.607 [2024/02/28 17:31:06.607 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:06.607 [2024/02/28 17:31:06.607 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=bb87475cc66aa5df754fd813cc358081] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:06.664 [2024/02/28 17:31:06.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626402] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626528386][leader=3]"] 2024-02-29 01:31:06.664 [2024/02/28 17:31:06.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626403] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:06.664 [2024/02/28 17:31:06.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626404] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626527785][leader=3]"] 2024-02-29 01:31:06.664 [2024/02/28 17:31:06.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626405] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:06.664 [2024/02/28 17:31:06.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626406] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:06.664 [2024/02/28 17:31:06.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626407] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:06.664 [2024/02/28 17:31:06.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626408] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626524214][leader=3]"] 2024-02-29 01:31:06.665 [2024/02/28 17:31:06.665 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626527784] [error="node offline[node=8]"] 2024-02-29 01:31:06.665 [2024/02/28 17:31:06.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:06.665 [2024/02/28 17:31:06.665 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:06.665 [2024/02/28 17:31:06.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626547521] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626525234] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626527784] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626553190] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:06.666 [2024/02/28 17:31:06.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:06.673 [2024/02/28 17:31:06.673 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626403] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:06.673 [2024/02/28 17:31:06.673 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626403] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:06.673 [2024/02/28 17:31:06.673 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626403] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:06.673 [2024/02/28 17:31:06.673 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626407] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:06.676 [2024/02/28 17:31:06.673 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626407] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:06.676 [2024/02/28 17:31:06.673 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626407] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:06.676 [2024/02/28 17:31:06.673 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626406] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:06.676 [2024/02/28 17:31:06.673 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626406] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:06.676 [2024/02/28 17:31:06.674 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626406] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:06.676 [2024/02/28 17:31:06.674 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626405] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:06.676 [2024/02/28 17:31:06.674 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626405] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:06.676 [2024/02/28 17:31:06.674 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626405] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:06.676 [2024/02/28 17:31:06.674 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626403] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:06.676 [2024/02/28 17:31:06.674 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626407] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:06.676 [2024/02/28 17:31:06.674 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626406] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:06.676 [2024/02/28 17:31:06.674 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626405] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:06.676 [2024/02/28 17:31:06.674 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=3] [toRemoveNum=4] 2024-02-29 01:31:06.676 [2024/02/28 17:31:06.674 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=43] [waitingTaskNum=0] [segmentTaskNum=43] [channelTaskNum=0] 2024-02-29 01:31:06.676 [2024/02/28 17:31:06.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626404] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:06.676 [2024/02/28 17:31:06.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626402] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:06.677 [2024/02/28 17:31:06.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626408] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:06.701 [2024/02/28 17:31:06.701 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=43] ["time cost"=25.29152ms] 2024-02-29 01:31:06.701 [2024/02/28 17:31:06.701 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=43] [committedNum=3] [toRemoveNum=0] 2024-02-29 01:31:06.701 [2024/02/28 17:31:06.701 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=43] [waitingTaskNum=0] [segmentTaskNum=43] [channelTaskNum=0] 2024-02-29 01:31:06.726 [2024/02/28 17:31:06.725 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626380] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:06.726 [2024/02/28 17:31:06.726 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626380] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:06.743 [2024/02/28 17:31:06.743 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:06.743 [2024/02/28 17:31:06.743 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=84a39e68e99ec7078b011e7408687d74] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:06.786 [2024/02/28 17:31:06.786 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626387] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626387] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626379] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626378] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626379] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626378] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626358] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626372] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626372] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626358] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626365] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626365] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626356] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626369] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626356] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626364] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626364] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626373] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626369] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626373] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626362] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626362] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626361] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626361] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626367] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626367] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.786 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626386] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.787 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626371] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.787 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626386] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.787 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626371] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.787 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626377] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.787 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626374] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.787 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626375] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.787 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626374] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.787 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626377] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.787 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626375] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:06.787 [2024/02/28 17:31:06.787 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626363] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626357] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626363] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626357] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626376] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626376] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626370] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626370] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626388] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626368] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626368] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626359] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626388] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626354] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626359] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626354] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626389] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626389] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626355] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:06.788 [2024/02/28 17:31:06.787 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626355] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:06.796 [2024/02/28 17:31:06.796 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626366] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:06.796 [2024/02/28 17:31:06.796 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626366] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:06.824 [2024/02/28 17:31:06.824 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626385] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:06.824 [2024/02/28 17:31:06.824 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626385] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:06.900 [2024/02/28 17:31:06.900 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626360] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:06.900 [2024/02/28 17:31:06.900 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626360] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:06.955 [2024/02/28 17:31:06.955 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=4b7c63634ae94ef68c9d87e54b154b0d] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298948,448039877626298981,448039877626298993,448039877626299022,448039877626299027,448039877626298936,448039877626298987,448039877626299011,448039877626299033,448039877626298953,448039877626299016,448039877626298970,448039877626298942,448039877626298965,448039877626299005,448039877626298976,448039877626298931,448039877626298959,448039877626298999]"] 2024-02-29 01:31:06.955 [2024/02/28 17:31:06.955 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=4b7c63634ae94ef68c9d87e54b154b0d] [collectionID=448039877626298924] 2024-02-29 01:31:06.955 [2024/02/28 17:31:06.955 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=4b7c63634ae94ef68c9d87e54b154b0d] [collectionID=448039877626298924] 2024-02-29 01:31:06.955 [2024/02/28 17:31:06.955 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:06.956 [2024/02/28 17:31:06.956 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=4b7c63634ae94ef68c9d87e54b154b0d] [collectionID=448039877626298924] 2024-02-29 01:31:06.956 [2024/02/28 17:31:06.956 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=4b7c63634ae94ef68c9d87e54b154b0d] [collectionID=448039877626298924] 2024-02-29 01:31:06.961 [2024/02/28 17:31:06.961 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=5ee2aeea8fc333743069fd9552f159c3] [collections="[448039877626298924]"] 2024-02-29 01:31:07.126 [2024/02/28 17:31:07.126 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626408] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:07.170 [2024/02/28 17:31:07.170 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=43] [waitingTaskNum=0] [segmentTaskNum=43] [channelTaskNum=0] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.171 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=11] ["time cost"=397.378µs] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.171 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626385] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.171 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.171 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626385] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760430] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.171 +00:00] [WARN] [task/executor.go:468] ["failed to sync distribution"] [taskID=1709136626408] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.171 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626408] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.171 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626372] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.171 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626372] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761556] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626363] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626363] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761031] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626356] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626356] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760757] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626359] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626359] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760744] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626360] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626360] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760570] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626368] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626368] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761143] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626387] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626387] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760422] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626377] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626377] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760215] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626358] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626358] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759971] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626354] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626354] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761400] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626362] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626362] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759628] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626357] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626357] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759725] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626369] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626369] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760192] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626371] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626371] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759826] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626378] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626378] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760374] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626380] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626380] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742592] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626389] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626389] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759968] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626386] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626386] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760748] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626373] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626373] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759770] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626361] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626361] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761482] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626365] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626365] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761308] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626367] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626367] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761572] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626375] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626375] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759849] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626355] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626355] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760025] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626370] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626370] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760696] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626364] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626364] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760970] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626374] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626374] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759612] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626379] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626379] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742556] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626376] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:07.175 [2024/02/28 17:31:07.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:07.176 [2024/02/28 17:31:07.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626376] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759777] 2024-02-29 01:31:07.176 [2024/02/28 17:31:07.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626388] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:07.176 [2024/02/28 17:31:07.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:07.176 [2024/02/28 17:31:07.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626388] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760199] 2024-02-29 01:31:07.176 [2024/02/28 17:31:07.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626366] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:07.176 [2024/02/28 17:31:07.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:07.176 [2024/02/28 17:31:07.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626366] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760366] 2024-02-29 01:31:07.176 [2024/02/28 17:31:07.174 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=11] [committedNum=0] [toRemoveNum=32] 2024-02-29 01:31:07.176 [2024/02/28 17:31:07.174 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=11] [waitingTaskNum=0] [segmentTaskNum=11] [channelTaskNum=0] 2024-02-29 01:31:07.215 [2024/02/28 17:31:07.215 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626404] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:07.216 [2024/02/28 17:31:07.216 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626402] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:07.218 [2024/02/28 17:31:07.218 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626404] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] [elapsed=2.922541ms] 2024-02-29 01:31:07.219 [2024/02/28 17:31:07.219 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626402] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] [elapsed=3.02962ms] 2024-02-29 01:31:07.247 [2024/02/28 17:31:07.246 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=65bf42692146b7eb812dfaee87d85274] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626299005,448039877626298993,448039877626298936,448039877626299011,448039877626299027,448039877626298948,448039877626298987,448039877626298942,448039877626299033,448039877626299022,448039877626298976,448039877626298965,448039877626298981,448039877626298931,448039877626298970,448039877626299016,448039877626298953,448039877626298999,448039877626298959]"] 2024-02-29 01:31:07.247 [2024/02/28 17:31:07.247 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=65bf42692146b7eb812dfaee87d85274] [collectionID=448039877626298924] 2024-02-29 01:31:07.247 [2024/02/28 17:31:07.247 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=65bf42692146b7eb812dfaee87d85274] [collectionID=448039877626298924] 2024-02-29 01:31:07.247 [2024/02/28 17:31:07.247 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:07.247 [2024/02/28 17:31:07.247 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=65bf42692146b7eb812dfaee87d85274] [collectionID=448039877626298924] 2024-02-29 01:31:07.247 [2024/02/28 17:31:07.247 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=65bf42692146b7eb812dfaee87d85274] [collectionID=448039877626298924] 2024-02-29 01:31:07.260 [2024/02/28 17:31:07.260 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=a4e5f55129b5090df349285c6e9fa636] [collections="[448039877626298924]"] 2024-02-29 01:31:07.322 [2024/02/28 17:31:07.322 +00:00] [INFO] [task/executor.go:211] ["load segments done"] [taskID=1709136625263] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626553190] [node=3] [source=segment_checker] [shardLeader=3] [elapsed=24.518866771s] 2024-02-29 01:31:07.610 [2024/02/28 17:31:07.610 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=b62fc6b94f2cad970a6e65318bc25d94] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626299022,448039877626299016,448039877626299011,448039877626298942,448039877626298936,448039877626298948,448039877626299033,448039877626298953,448039877626299005,448039877626298981,448039877626298993,448039877626298970,448039877626298999,448039877626298931,448039877626298976,448039877626299027,448039877626298987,448039877626298965,448039877626298959]"] 2024-02-29 01:31:07.610 [2024/02/28 17:31:07.610 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=b62fc6b94f2cad970a6e65318bc25d94] [collectionID=448039877626298924] 2024-02-29 01:31:07.610 [2024/02/28 17:31:07.610 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=b62fc6b94f2cad970a6e65318bc25d94] [collectionID=448039877626298924] 2024-02-29 01:31:07.610 [2024/02/28 17:31:07.610 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:07.611 [2024/02/28 17:31:07.611 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=b62fc6b94f2cad970a6e65318bc25d94] [collectionID=448039877626298924] 2024-02-29 01:31:07.611 [2024/02/28 17:31:07.611 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=b62fc6b94f2cad970a6e65318bc25d94] [collectionID=448039877626298924] 2024-02-29 01:31:07.617 [2024/02/28 17:31:07.617 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=ddb271f3b92dbe3cd0f73082f9bde9ad] [collections="[448039877626298924]"] 2024-02-29 01:31:07.643 [2024/02/28 17:31:07.643 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=56dabdd7d1a5ac3c0d6623ac5c6efbd2] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298931,448039877626299022,448039877626298948,448039877626298976,448039877626298965,448039877626298959,448039877626298999,448039877626299033,448039877626299011,448039877626299016,448039877626298936,448039877626298953,448039877626298970,448039877626299005,448039877626299027,448039877626298981,448039877626298987,448039877626298993,448039877626298942]"] 2024-02-29 01:31:07.643 [2024/02/28 17:31:07.643 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=56dabdd7d1a5ac3c0d6623ac5c6efbd2] [collectionID=448039877626298924] 2024-02-29 01:31:07.643 [2024/02/28 17:31:07.643 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=56dabdd7d1a5ac3c0d6623ac5c6efbd2] [collectionID=448039877626298924] 2024-02-29 01:31:07.643 [2024/02/28 17:31:07.643 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:07.644 [2024/02/28 17:31:07.644 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=56dabdd7d1a5ac3c0d6623ac5c6efbd2] [collectionID=448039877626298924] 2024-02-29 01:31:07.644 [2024/02/28 17:31:07.644 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=56dabdd7d1a5ac3c0d6623ac5c6efbd2] [collectionID=448039877626298924] 2024-02-29 01:31:07.663 [2024/02/28 17:31:07.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626409] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:07.663 [2024/02/28 17:31:07.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626410] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:07.663 [2024/02/28 17:31:07.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626411] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:07.663 [2024/02/28 17:31:07.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626412] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:07.664 [2024/02/28 17:31:07.664 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626527182] 2024-02-29 01:31:07.664 [2024/02/28 17:31:07.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626527182]"] 2024-02-29 01:31:07.664 [2024/02/28 17:31:07.664 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626525234] [error="node offline[node=8]"] 2024-02-29 01:31:07.664 [2024/02/28 17:31:07.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626547521] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626525234] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626527784] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626553190] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:07.665 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=c1c78817a4fb5741022508c1a810d3e3] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626299016,448039877626298993,448039877626298931,448039877626299027,448039877626299022,448039877626298987,448039877626298965,448039877626298981,448039877626298942,448039877626298948,448039877626299033,448039877626299011,448039877626299005,448039877626298959,448039877626298970,448039877626298953,448039877626298936,448039877626298976,448039877626298999]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=c1c78817a4fb5741022508c1a810d3e3] [collectionID=448039877626298924] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=c1c78817a4fb5741022508c1a810d3e3] [collectionID=448039877626298924] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626413] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761400]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626414] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760025]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626415] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760757]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626416] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760430]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626417] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759725]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626418] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759971]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626419] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760744]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626420] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760570]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626421] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761482]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626422] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759628]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626423] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761031]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626424] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760970]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626425] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761308]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626426] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760748]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626427] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760366]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626428] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761572]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626429] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760192]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626430] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761143]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626431] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760422]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626432] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760696]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626433] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761556]"] 2024-02-29 01:31:07.666 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626434] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759826]"] 2024-02-29 01:31:07.667 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626435] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759612]"] 2024-02-29 01:31:07.667 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626436] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759770]"] 2024-02-29 01:31:07.667 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626437] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759849]"] 2024-02-29 01:31:07.667 [2024/02/28 17:31:07.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626438] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759777]"] 2024-02-29 01:31:07.667 [2024/02/28 17:31:07.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626439] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760215]"] 2024-02-29 01:31:07.667 [2024/02/28 17:31:07.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626440] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760199]"] 2024-02-29 01:31:07.667 [2024/02/28 17:31:07.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626441] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760374]"] 2024-02-29 01:31:07.667 [2024/02/28 17:31:07.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626442] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759968]"] 2024-02-29 01:31:07.667 [2024/02/28 17:31:07.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626443] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742556]"] 2024-02-29 01:31:07.667 [2024/02/28 17:31:07.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626444] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742592]"] 2024-02-29 01:31:07.667 [2024/02/28 17:31:07.667 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=c1c78817a4fb5741022508c1a810d3e3] [collectionID=448039877626298924] 2024-02-29 01:31:07.667 [2024/02/28 17:31:07.667 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=c1c78817a4fb5741022508c1a810d3e3] [collectionID=448039877626298924] 2024-02-29 01:31:07.674 [2024/02/28 17:31:07.674 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626412] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:07.674 [2024/02/28 17:31:07.674 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626412] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:07.674 [2024/02/28 17:31:07.674 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626412] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:07.674 [2024/02/28 17:31:07.674 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626411] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:07.674 [2024/02/28 17:31:07.674 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626411] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:07.674 [2024/02/28 17:31:07.674 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626411] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:07.674 [2024/02/28 17:31:07.674 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626409] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:07.675 [2024/02/28 17:31:07.674 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626409] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:07.675 [2024/02/28 17:31:07.675 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626409] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:07.675 [2024/02/28 17:31:07.675 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626410] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:07.675 [2024/02/28 17:31:07.675 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626410] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:07.675 [2024/02/28 17:31:07.675 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626410] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:07.676 [2024/02/28 17:31:07.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626412] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:07.676 [2024/02/28 17:31:07.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626411] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:07.676 [2024/02/28 17:31:07.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626409] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:07.676 [2024/02/28 17:31:07.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626410] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:07.676 [2024/02/28 17:31:07.676 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=32] [toRemoveNum=4] 2024-02-29 01:31:07.676 [2024/02/28 17:31:07.676 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=43] [waitingTaskNum=0] [segmentTaskNum=43] [channelTaskNum=0] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626431] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626426] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626438] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626415] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626425] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626444] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626428] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626422] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626433] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=39] ["time cost"=490.251µs] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626427] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626408] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626524214] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626402] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626528386] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626434] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626421] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626429] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626435] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626417] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626404] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626527785] 2024-02-29 01:31:07.677 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626430] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626436] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626423] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626418] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626442] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626440] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626416] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626441] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626439] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626424] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626420] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.677 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136625263] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626553190] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.678 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=39] [committedNum=32] [toRemoveNum=4] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626432] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626443] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.678 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=39] [waitingTaskNum=0] [segmentTaskNum=39] [channelTaskNum=0] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626413] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626419] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626414] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.678 [2024/02/28 17:31:07.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626437] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:07.680 [2024/02/28 17:31:07.679 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=b2d6983b3feb638c9c2c590ba49950fd] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298987,448039877626299005,448039877626298970,448039877626298965,448039877626299033,448039877626298993,448039877626298936,448039877626298976,448039877626299011,448039877626299022,448039877626298999,448039877626298959,448039877626298931,448039877626299016,448039877626298953,448039877626298981,448039877626298942,448039877626299027,448039877626298948]"] 2024-02-29 01:31:07.680 [2024/02/28 17:31:07.680 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=b2d6983b3feb638c9c2c590ba49950fd] [collectionID=448039877626298924] 2024-02-29 01:31:07.680 [2024/02/28 17:31:07.680 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=b2d6983b3feb638c9c2c590ba49950fd] [collectionID=448039877626298924] 2024-02-29 01:31:07.680 [2024/02/28 17:31:07.680 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:07.684 [2024/02/28 17:31:07.684 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=b2d6983b3feb638c9c2c590ba49950fd] [collectionID=448039877626298924] 2024-02-29 01:31:07.685 [2024/02/28 17:31:07.684 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=b2d6983b3feb638c9c2c590ba49950fd] [collectionID=448039877626298924] 2024-02-29 01:31:07.728 [2024/02/28 17:31:07.728 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626444] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.731 [2024/02/28 17:31:07.731 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626437] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.733 [2024/02/28 17:31:07.733 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626425] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.734 [2024/02/28 17:31:07.734 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626419] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.734 [2024/02/28 17:31:07.734 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=e7ec280bda0b7f9e1e84f32c4b5b2171] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298942,448039877626298959,448039877626298948,448039877626298987,448039877626298981,448039877626299016,448039877626298965,448039877626298976,448039877626299005,448039877626298931,448039877626299033,448039877626299027,448039877626298993,448039877626298953,448039877626299011,448039877626298936,448039877626298999,448039877626299022,448039877626298970]"] 2024-02-29 01:31:07.734 [2024/02/28 17:31:07.734 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=e7ec280bda0b7f9e1e84f32c4b5b2171] [collectionID=448039877626298924] 2024-02-29 01:31:07.734 [2024/02/28 17:31:07.734 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=e7ec280bda0b7f9e1e84f32c4b5b2171] [collectionID=448039877626298924] 2024-02-29 01:31:07.734 [2024/02/28 17:31:07.734 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:07.738 [2024/02/28 17:31:07.737 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=e7ec280bda0b7f9e1e84f32c4b5b2171] [collectionID=448039877626298924] 2024-02-29 01:31:07.738 [2024/02/28 17:31:07.737 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=e7ec280bda0b7f9e1e84f32c4b5b2171] [collectionID=448039877626298924] 2024-02-29 01:31:07.740 [2024/02/28 17:31:07.740 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626432] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.740 [2024/02/28 17:31:07.740 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626427] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.740 [2024/02/28 17:31:07.740 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626440] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.740 [2024/02/28 17:31:07.740 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626413] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.741 [2024/02/28 17:31:07.741 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626418] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.742 [2024/02/28 17:31:07.741 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626420] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.742 [2024/02/28 17:31:07.742 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626426] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.742 [2024/02/28 17:31:07.742 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626424] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.743 [2024/02/28 17:31:07.743 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626431] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.743 [2024/02/28 17:31:07.743 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626421] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.746 [2024/02/28 17:31:07.746 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626415] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.747 [2024/02/28 17:31:07.747 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626416] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.747 [2024/02/28 17:31:07.747 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626436] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.748 [2024/02/28 17:31:07.748 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626439] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.748 [2024/02/28 17:31:07.748 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626434] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.748 [2024/02/28 17:31:07.748 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626429] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.748 [2024/02/28 17:31:07.748 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626438] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.749 [2024/02/28 17:31:07.749 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626428] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.751 [2024/02/28 17:31:07.751 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626443] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.751 [2024/02/28 17:31:07.751 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626441] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.751 [2024/02/28 17:31:07.751 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626435] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.751 [2024/02/28 17:31:07.751 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626442] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.751 [2024/02/28 17:31:07.751 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626422] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.751 [2024/02/28 17:31:07.751 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626417] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.751 [2024/02/28 17:31:07.751 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626430] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.751 [2024/02/28 17:31:07.751 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626423] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.751 [2024/02/28 17:31:07.751 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626414] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.752 [2024/02/28 17:31:07.752 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626433] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:07.758 [2024/02/28 17:31:07.758 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=d6a5d24f849dd7639f6f9a2daec6046a] [collections="[448039877626298924]"] 2024-02-29 01:31:07.758 [2024/02/28 17:31:07.758 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=f6a9a41b64f55ac74f3075bf89aa4c63] [collections="[448039877626298924]"] 2024-02-29 01:31:07.758 [2024/02/28 17:31:07.758 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=56326e3a697d8dab16e917f32a86f0b6] [collections="[448039877626298924]"] 2024-02-29 01:31:07.759 [2024/02/28 17:31:07.759 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626527784] [error="node offline[node=8]"] 2024-02-29 01:31:07.759 [2024/02/28 17:31:07.759 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_1_448039877626298924v1 is not available in any replica"] [traceID=e16624518bf3fbcd8a16bd717ea4ab86] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:07.759 [2024/02/28 17:31:07.759 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=ff555f46f910aa5549dac400c4176022] [collections="[448039877626298924]"] 2024-02-29 01:31:07.761 [2024/02/28 17:31:07.761 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=693d46bdb5d6711af606dcd8b6af2e38] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626299005,448039877626298931,448039877626298987,448039877626298948,448039877626298942,448039877626298999,448039877626299027,448039877626299011,448039877626298953,448039877626299016,448039877626298976,448039877626299033,448039877626298981,448039877626298965,448039877626298993,448039877626298936,448039877626298970,448039877626298959,448039877626299022]"] 2024-02-29 01:31:07.761 [2024/02/28 17:31:07.761 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=693d46bdb5d6711af606dcd8b6af2e38] [collectionID=448039877626298924] 2024-02-29 01:31:07.761 [2024/02/28 17:31:07.761 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=693d46bdb5d6711af606dcd8b6af2e38] [collectionID=448039877626298924] 2024-02-29 01:31:07.761 [2024/02/28 17:31:07.761 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:07.762 [2024/02/28 17:31:07.762 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=693d46bdb5d6711af606dcd8b6af2e38] [collectionID=448039877626298924] 2024-02-29 01:31:07.762 [2024/02/28 17:31:07.762 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=693d46bdb5d6711af606dcd8b6af2e38] [collectionID=448039877626298924] 2024-02-29 01:31:07.764 [2024/02/28 17:31:07.764 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:07.765 [2024/02/28 17:31:07.764 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=c782bd2f524799a5ad01f7fefb30cd06] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:07.765 [2024/02/28 17:31:07.765 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626527182] 2024-02-29 01:31:07.765 [2024/02/28 17:31:07.765 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=6750ee4273acf3558eecf2f03fd76246] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626527182]"] 2024-02-29 01:31:07.769 [2024/02/28 17:31:07.769 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:07.769 [2024/02/28 17:31:07.769 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=15db7c6816a21e438c5d8b87d3e00347] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:07.775 [2024/02/28 17:31:07.775 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:07.775 [2024/02/28 17:31:07.775 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=19f546683f9a184269cae04891708204] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:07.786 [2024/02/28 17:31:07.785 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:07.786 [2024/02/28 17:31:07.786 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=a3e06209424d8804e261f329c953dcba] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:07.792 [2024/02/28 17:31:07.792 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:07.792 [2024/02/28 17:31:07.792 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=787a8dfa196dbab2290f312e6da72507] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:07.797 [2024/02/28 17:31:07.797 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=b8f3f449e7375d238718e1b142a8ebce] [collections="[448039877626298924]"] 2024-02-29 01:31:07.800 [2024/02/28 17:31:07.800 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:07.801 [2024/02/28 17:31:07.800 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=0a76afd034b699e44d61b72ecb255113] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:07.804 [2024/02/28 17:31:07.804 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=7cf1b758e387a84ac8aeda01a1797cde] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298965,448039877626299022,448039877626298936,448039877626298970,448039877626298948,448039877626298959,448039877626298999,448039877626298931,448039877626298942,448039877626299011,448039877626298976,448039877626299016,448039877626299033,448039877626298993,448039877626299005,448039877626299027,448039877626298987,448039877626298981,448039877626298953]"] 2024-02-29 01:31:07.804 [2024/02/28 17:31:07.804 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=7cf1b758e387a84ac8aeda01a1797cde] [collectionID=448039877626298924] 2024-02-29 01:31:07.804 [2024/02/28 17:31:07.804 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=7cf1b758e387a84ac8aeda01a1797cde] [collectionID=448039877626298924] 2024-02-29 01:31:07.804 [2024/02/28 17:31:07.804 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:07.805 [2024/02/28 17:31:07.805 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=7cf1b758e387a84ac8aeda01a1797cde] [collectionID=448039877626298924] 2024-02-29 01:31:07.805 [2024/02/28 17:31:07.805 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=7cf1b758e387a84ac8aeda01a1797cde] [collectionID=448039877626298924] 2024-02-29 01:31:07.831 [2024/02/28 17:31:07.830 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=7488e978fcf3cc9c9638cfab6c83835b] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298987,448039877626298970,448039877626299027,448039877626298931,448039877626299033,448039877626298953,448039877626298936,448039877626298965,448039877626299005,448039877626298942,448039877626298976,448039877626299022,448039877626299011,448039877626299016,448039877626298993,448039877626298981,448039877626298959,448039877626298948,448039877626298999]"] 2024-02-29 01:31:07.833 [2024/02/28 17:31:07.832 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=7488e978fcf3cc9c9638cfab6c83835b] [collectionID=448039877626298924] 2024-02-29 01:31:07.833 [2024/02/28 17:31:07.832 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=7488e978fcf3cc9c9638cfab6c83835b] [collectionID=448039877626298924] 2024-02-29 01:31:07.833 [2024/02/28 17:31:07.832 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:07.834 [2024/02/28 17:31:07.834 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=7488e978fcf3cc9c9638cfab6c83835b] [collectionID=448039877626298924] 2024-02-29 01:31:07.834 [2024/02/28 17:31:07.834 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=7488e978fcf3cc9c9638cfab6c83835b] [collectionID=448039877626298924] 2024-02-29 01:31:07.848 [2024/02/28 17:31:07.848 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=2da7fdb021a19f175360ff7a08668918] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298970,448039877626298981,448039877626299011,448039877626298936,448039877626299016,448039877626298965,448039877626299033,448039877626298948,448039877626298942,448039877626299027,448039877626298953,448039877626298976,448039877626298993,448039877626299005,448039877626298931,448039877626298987,448039877626298999,448039877626298959,448039877626299022]"] 2024-02-29 01:31:07.848 [2024/02/28 17:31:07.848 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=2da7fdb021a19f175360ff7a08668918] [collectionID=448039877626298924] 2024-02-29 01:31:07.848 [2024/02/28 17:31:07.848 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=2da7fdb021a19f175360ff7a08668918] [collectionID=448039877626298924] 2024-02-29 01:31:07.848 [2024/02/28 17:31:07.848 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:07.849 [2024/02/28 17:31:07.849 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=2da7fdb021a19f175360ff7a08668918] [collectionID=448039877626298924] 2024-02-29 01:31:07.849 [2024/02/28 17:31:07.849 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=2da7fdb021a19f175360ff7a08668918] [collectionID=448039877626298924] 2024-02-29 01:31:07.882 [2024/02/28 17:31:07.881 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=0e957ced54e811604ab27309d16a8890] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298936,448039877626298993,448039877626299033,448039877626298959,448039877626299005,448039877626299027,448039877626298942,448039877626298931,448039877626298981,448039877626299022,448039877626298948,448039877626298953,448039877626298965,448039877626298999,448039877626298970,448039877626299016,448039877626298987,448039877626298976,448039877626299011]"] 2024-02-29 01:31:07.882 [2024/02/28 17:31:07.882 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=0e957ced54e811604ab27309d16a8890] [collectionID=448039877626298924] 2024-02-29 01:31:07.882 [2024/02/28 17:31:07.882 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=0e957ced54e811604ab27309d16a8890] [collectionID=448039877626298924] 2024-02-29 01:31:07.882 [2024/02/28 17:31:07.882 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:07.888 [2024/02/28 17:31:07.887 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=0e957ced54e811604ab27309d16a8890] [collectionID=448039877626298924] 2024-02-29 01:31:07.888 [2024/02/28 17:31:07.887 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=0e957ced54e811604ab27309d16a8890] [collectionID=448039877626298924] 2024-02-29 01:31:08.024 [2024/02/28 17:31:08.023 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=6e5d90c4224ec5fe13d670d00b5a8cb7] [collections="[448039877626298924]"] 2024-02-29 01:31:08.024 [2024/02/28 17:31:08.023 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:08.025 [2024/02/28 17:31:08.023 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=cc07de0de85853a2569df1680b6bb785] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:08.025 [2024/02/28 17:31:08.023 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=bce6297fad9c0e19bfe3cce10947976d] [collections="[448039877626298924]"] 2024-02-29 01:31:08.025 [2024/02/28 17:31:08.023 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=42bcd93f9490b7e27e8f69e93370892e] [collections="[448039877626298924]"] 2024-02-29 01:31:08.086 [2024/02/28 17:31:08.084 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=fb3043984a5b011e5b58f54280fbed32] [collections="[448039877626298924]"] 2024-02-29 01:31:08.091 [2024/02/28 17:31:08.091 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626527784] [error="node offline[node=8]"] 2024-02-29 01:31:08.091 [2024/02/28 17:31:08.091 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_1_448039877626298924v1 is not available in any replica"] [traceID=53f293a44ce0909d069f43719fd70a0f] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:08.099 [2024/02/28 17:31:08.099 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:08.099 [2024/02/28 17:31:08.099 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=ac3b9d0b789b666788cbcc729b62c274] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:08.196 [2024/02/28 17:31:08.193 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=39] [waitingTaskNum=0] [segmentTaskNum=39] [channelTaskNum=0] 2024-02-29 01:31:08.197 [2024/02/28 17:31:08.197 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=39] ["time cost"=280.547µs] 2024-02-29 01:31:08.197 [2024/02/28 17:31:08.197 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=39] [committedNum=0] [toRemoveNum=0] 2024-02-29 01:31:08.197 [2024/02/28 17:31:08.197 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=39] [waitingTaskNum=0] [segmentTaskNum=39] [channelTaskNum=0] 2024-02-29 01:31:08.280 [2024/02/28 17:31:08.271 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:08.280 [2024/02/28 17:31:08.272 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=d9f629c7646daec249a49ed6d8126233] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:08.380 [2024/02/28 17:31:08.380 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:08.382 [2024/02/28 17:31:08.381 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=856cd4a2b9bcde4ac61a679e426e4b53] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:08.495 [2024/02/28 17:31:08.495 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:08.495 [2024/02/28 17:31:08.495 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=72a4dd31cfb16098f3b50b602784aa8d] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:08.503 [2024/02/28 17:31:08.502 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:08.503 [2024/02/28 17:31:08.502 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=16c13d749c78bf3ba219aad8bcb22092] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:08.663 [2024/02/28 17:31:08.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626445] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:08.663 [2024/02/28 17:31:08.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626446] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:08.663 [2024/02/28 17:31:08.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626447] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:08.663 [2024/02/28 17:31:08.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626448] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626524214][leader=3]"] 2024-02-29 01:31:08.663 [2024/02/28 17:31:08.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626449] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:08.663 [2024/02/28 17:31:08.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626450] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626528386][leader=3]"] 2024-02-29 01:31:08.663 [2024/02/28 17:31:08.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626451] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626527785][leader=3]"] 2024-02-29 01:31:08.664 [2024/02/28 17:31:08.664 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:08.664 [2024/02/28 17:31:08.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:08.664 [2024/02/28 17:31:08.664 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:08.664 [2024/02/28 17:31:08.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626547521] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626525234] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626527784] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:08.665 [2024/02/28 17:31:08.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:08.670 [2024/02/28 17:31:08.670 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626446] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:08.670 [2024/02/28 17:31:08.670 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626446] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:08.673 [2024/02/28 17:31:08.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626446] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:08.673 [2024/02/28 17:31:08.670 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626447] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:08.673 [2024/02/28 17:31:08.670 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626447] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:08.673 [2024/02/28 17:31:08.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626447] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:08.673 [2024/02/28 17:31:08.671 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626445] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:08.673 [2024/02/28 17:31:08.671 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626445] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:08.673 [2024/02/28 17:31:08.671 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626445] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:08.673 [2024/02/28 17:31:08.671 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626449] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:08.673 [2024/02/28 17:31:08.671 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626449] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:08.673 [2024/02/28 17:31:08.671 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626449] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:08.673 [2024/02/28 17:31:08.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626446] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:08.673 [2024/02/28 17:31:08.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626447] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:08.673 [2024/02/28 17:31:08.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626445] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:08.673 [2024/02/28 17:31:08.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626449] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:08.673 [2024/02/28 17:31:08.671 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=3] [toRemoveNum=4] 2024-02-29 01:31:08.673 [2024/02/28 17:31:08.671 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=42] [waitingTaskNum=0] [segmentTaskNum=42] [channelTaskNum=0] 2024-02-29 01:31:08.675 [2024/02/28 17:31:08.675 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=42] ["time cost"=1.036262ms] 2024-02-29 01:31:08.675 [2024/02/28 17:31:08.675 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=42] [committedNum=3] [toRemoveNum=0] 2024-02-29 01:31:08.675 [2024/02/28 17:31:08.675 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=42] [waitingTaskNum=0] [segmentTaskNum=42] [channelTaskNum=0] 2024-02-29 01:31:08.675 [2024/02/28 17:31:08.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626451] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:08.676 [2024/02/28 17:31:08.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626448] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:08.676 [2024/02/28 17:31:08.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626450] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:09.112 [2024/02/28 17:31:09.112 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=9521bf57ce3b4fb997a818151a204af4] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626299027,448039877626299016,448039877626298976,448039877626298993,448039877626298936,448039877626298999,448039877626299011,448039877626298953,448039877626299022,448039877626298942,448039877626298981,448039877626298959,448039877626298970,448039877626299033,448039877626299005,448039877626298931,448039877626298987,448039877626298948,448039877626298965]"] 2024-02-29 01:31:09.112 [2024/02/28 17:31:09.112 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=9521bf57ce3b4fb997a818151a204af4] [collectionID=448039877626298924] 2024-02-29 01:31:09.112 [2024/02/28 17:31:09.112 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=9521bf57ce3b4fb997a818151a204af4] [collectionID=448039877626298924] 2024-02-29 01:31:09.112 [2024/02/28 17:31:09.112 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:09.113 [2024/02/28 17:31:09.113 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=9521bf57ce3b4fb997a818151a204af4] [collectionID=448039877626298924] 2024-02-29 01:31:09.113 [2024/02/28 17:31:09.113 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=9521bf57ce3b4fb997a818151a204af4] [collectionID=448039877626298924] 2024-02-29 01:31:09.118 [2024/02/28 17:31:09.118 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=3b6cbc11de736103387f0620be63a715] [collections="[448039877626298924]"] 2024-02-29 01:31:09.169 [2024/02/28 17:31:09.169 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=42] [waitingTaskNum=0] [segmentTaskNum=42] [channelTaskNum=0] 2024-02-29 01:31:09.170 [2024/02/28 17:31:09.170 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=42] ["time cost"=233.224µs] 2024-02-29 01:31:09.170 [2024/02/28 17:31:09.170 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=42] [committedNum=0] [toRemoveNum=0] 2024-02-29 01:31:09.170 [2024/02/28 17:31:09.170 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=42] [waitingTaskNum=0] [segmentTaskNum=42] [channelTaskNum=0] 2024-02-29 01:31:09.191 [2024/02/28 17:31:09.190 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626448] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:09.196 [2024/02/28 17:31:09.192 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626450] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:09.198 [2024/02/28 17:31:09.197 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626450] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] [elapsed=5.430052ms] 2024-02-29 01:31:09.307 [2024/02/28 17:31:09.307 +00:00] [WARN] [task/executor.go:468] ["failed to sync distribution"] [taskID=1709136626448] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:09.307 [2024/02/28 17:31:09.307 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626448] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:09.398 [2024/02/28 17:31:09.398 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626451] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:09.401 [2024/02/28 17:31:09.401 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626451] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] [elapsed=3.572273ms] 2024-02-29 01:31:09.664 [2024/02/28 17:31:09.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626452] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:09.664 [2024/02/28 17:31:09.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626453] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:09.664 [2024/02/28 17:31:09.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626454] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:09.664 [2024/02/28 17:31:09.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626455] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:09.665 [2024/02/28 17:31:09.665 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:09.665 [2024/02/28 17:31:09.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:09.665 [2024/02/28 17:31:09.665 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:09.665 [2024/02/28 17:31:09.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:09.666 [2024/02/28 17:31:09.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626525234] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:09.666 [2024/02/28 17:31:09.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626527784] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:09.666 [2024/02/28 17:31:09.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:09.666 [2024/02/28 17:31:09.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:09.666 [2024/02/28 17:31:09.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:09.666 [2024/02/28 17:31:09.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:09.666 [2024/02/28 17:31:09.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.666 [2024/02/28 17:31:09.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.666 [2024/02/28 17:31:09.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626547521] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.666 [2024/02/28 17:31:09.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.667 [2024/02/28 17:31:09.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:09.699 [2024/02/28 17:31:09.698 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626455] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:09.699 [2024/02/28 17:31:09.698 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626455] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:09.699 [2024/02/28 17:31:09.699 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626455] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:09.699 [2024/02/28 17:31:09.699 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626452] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:09.699 [2024/02/28 17:31:09.699 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626452] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:09.699 [2024/02/28 17:31:09.699 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626452] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:09.699 [2024/02/28 17:31:09.699 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626453] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:09.699 [2024/02/28 17:31:09.699 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626453] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:09.699 [2024/02/28 17:31:09.699 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626453] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:09.699 [2024/02/28 17:31:09.699 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626454] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:09.699 [2024/02/28 17:31:09.699 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626454] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:09.700 [2024/02/28 17:31:09.699 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626454] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:09.700 [2024/02/28 17:31:09.700 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626455] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:09.700 [2024/02/28 17:31:09.700 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626452] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:09.700 [2024/02/28 17:31:09.700 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626453] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:09.700 [2024/02/28 17:31:09.700 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626454] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:09.700 [2024/02/28 17:31:09.700 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=0] [toRemoveNum=4] 2024-02-29 01:31:09.700 [2024/02/28 17:31:09.700 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=42] [waitingTaskNum=0] [segmentTaskNum=42] [channelTaskNum=0] 2024-02-29 01:31:09.701 [2024/02/28 17:31:09.701 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=39] ["time cost"=198.936µs] 2024-02-29 01:31:09.701 [2024/02/28 17:31:09.701 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626448] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626524214] 2024-02-29 01:31:09.701 [2024/02/28 17:31:09.701 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626450] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626528386] 2024-02-29 01:31:09.701 [2024/02/28 17:31:09.701 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626451] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626527785] 2024-02-29 01:31:09.701 [2024/02/28 17:31:09.701 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=39] [committedNum=0] [toRemoveNum=3] 2024-02-29 01:31:09.701 [2024/02/28 17:31:09.701 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=39] [waitingTaskNum=0] [segmentTaskNum=39] [channelTaskNum=0] 2024-02-29 01:31:09.703 [2024/02/28 17:31:09.703 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626419] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:09.703 [2024/02/28 17:31:09.703 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626419] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:09.703 [2024/02/28 17:31:09.703 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626444] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:09.703 [2024/02/28 17:31:09.703 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626444] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:09.703 [2024/02/28 17:31:09.703 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626429] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:09.703 [2024/02/28 17:31:09.703 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626429] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626427] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626442] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626424] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626442] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626424] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626431] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626431] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626439] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626427] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626415] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626439] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626415] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626435] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626435] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626423] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626423] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626430] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626436] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626436] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626430] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626428] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626428] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136625262] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626526417] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626526417/448039877626558818]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136625262] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626526417/448039877626558818]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626420] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626418] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626420] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626418] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626440] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626440] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626426] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626426] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626438] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:09.759 [2024/02/28 17:31:09.759 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626438] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626425] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626414] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626425] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626414] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626421] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626421] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626441] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626441] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626416] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626416] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626434] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626434] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626437] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626437] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626443] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626443] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626422] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626422] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:09.760 [2024/02/28 17:31:09.760 +00:00] [INFO] [task/executor.go:211] ["load segments done"] [taskID=1709136625260] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626525234] [node=3] [source=segment_checker] [shardLeader=3] [elapsed=26.961148255s] 2024-02-29 01:31:09.761 [2024/02/28 17:31:09.761 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626413] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:09.761 [2024/02/28 17:31:09.761 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626413] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:09.782 [2024/02/28 17:31:09.778 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626417] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:09.782 [2024/02/28 17:31:09.778 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626417] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:09.796 [2024/02/28 17:31:09.794 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626432] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:09.796 [2024/02/28 17:31:09.794 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626432] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:09.815 [2024/02/28 17:31:09.814 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=c7e9552e81ce600c267d7f43691807e9] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298987,448039877626299016,448039877626298999,448039877626298948,448039877626298993,448039877626299011,448039877626298981,448039877626299005,448039877626298970,448039877626299022,448039877626298965,448039877626298942,448039877626299033,448039877626298959,448039877626298953,448039877626298936,448039877626298976,448039877626298931,448039877626299027]"] 2024-02-29 01:31:09.815 [2024/02/28 17:31:09.815 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=c7e9552e81ce600c267d7f43691807e9] [collectionID=448039877626298924] 2024-02-29 01:31:09.815 [2024/02/28 17:31:09.815 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=c7e9552e81ce600c267d7f43691807e9] [collectionID=448039877626298924] 2024-02-29 01:31:09.815 [2024/02/28 17:31:09.815 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:09.815 [2024/02/28 17:31:09.815 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=c7e9552e81ce600c267d7f43691807e9] [collectionID=448039877626298924] 2024-02-29 01:31:09.815 [2024/02/28 17:31:09.815 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=c7e9552e81ce600c267d7f43691807e9] [collectionID=448039877626298924] 2024-02-29 01:31:09.821 [2024/02/28 17:31:09.821 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=dc41472ca8ab84bf44511de6a54c8378] [collections="[448039877626298924]"] 2024-02-29 01:31:09.826 [2024/02/28 17:31:09.825 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626433] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:09.826 [2024/02/28 17:31:09.826 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626433] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:10.078 [2024/02/28 17:31:10.078 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626525234] [error="node offline[node=8]"] 2024-02-29 01:31:10.079 [2024/02/28 17:31:10.078 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_1_448039877626298924v1 is not available in any replica"] [traceID=a0ff855e2b7ba6c97c92162a9ca92256] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:10.097 [2024/02/28 17:31:10.097 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:10.098 [2024/02/28 17:31:10.097 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=9f5c125f6cee47df7fa30d4d6fe20922] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:10.174 [2024/02/28 17:31:10.174 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=39] [waitingTaskNum=0] [segmentTaskNum=39] [channelTaskNum=0] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=5] ["time cost"=256.316µs] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626431] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626431] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760422] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626422] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626422] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759628] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626426] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626426] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760748] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626415] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626415] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760757] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626444] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626444] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742592] 2024-02-29 01:31:10.175 [2024/02/28 17:31:10.175 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626427] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.175 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.175 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626427] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760366] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.175 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626438] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.175 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.175 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626438] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759777] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626428] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626428] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761572] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626425] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626425] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761308] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626433] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626433] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761556] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626418] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626418] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759971] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626442] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626442] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759968] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626417] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626417] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759725] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626440] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:10.176 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626440] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760199] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626430] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626430] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761143] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626416] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626416] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760430] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626420] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626420] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760570] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626429] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626429] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760192] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626435] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626435] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759612] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626436] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626436] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759770] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626421] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626421] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761482] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626434] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626434] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759826] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626419] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626419] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760744] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626423] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626423] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761031] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626441] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626441] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760374] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626424] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626424] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760970] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626443] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626443] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742556] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626439] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626439] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760215] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626413] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626413] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761400] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626432] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626432] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760696] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626414] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626414] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760025] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136625260] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626525234] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626437] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626437] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759849] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136625262] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626526417] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626526417/448039877626558818]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626526417/448039877626558818]"] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136625262] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626526417] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=5] [committedNum=0] [toRemoveNum=34] 2024-02-29 01:31:10.177 [2024/02/28 17:31:10.177 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=5] [waitingTaskNum=0] [segmentTaskNum=5] [channelTaskNum=0] 2024-02-29 01:31:10.461 [2024/02/28 17:31:10.461 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=48152dda60e4deb49b4a60b242a22035] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298936,448039877626299033,448039877626299022,448039877626298965,448039877626298999,448039877626298953,448039877626299005,448039877626298970,448039877626298942,448039877626298976,448039877626299011,448039877626299016,448039877626298993,448039877626298987,448039877626299027,448039877626298959,448039877626298981,448039877626298948,448039877626298931]"] 2024-02-29 01:31:10.461 [2024/02/28 17:31:10.461 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=48152dda60e4deb49b4a60b242a22035] [collectionID=448039877626298924] 2024-02-29 01:31:10.461 [2024/02/28 17:31:10.461 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=48152dda60e4deb49b4a60b242a22035] [collectionID=448039877626298924] 2024-02-29 01:31:10.461 [2024/02/28 17:31:10.461 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:10.462 [2024/02/28 17:31:10.462 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=48152dda60e4deb49b4a60b242a22035] [collectionID=448039877626298924] 2024-02-29 01:31:10.462 [2024/02/28 17:31:10.462 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=48152dda60e4deb49b4a60b242a22035] [collectionID=448039877626298924] 2024-02-29 01:31:10.467 [2024/02/28 17:31:10.467 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=052ee85d6094a1b017d299f3a322671d] [collections="[448039877626298924]"] 2024-02-29 01:31:10.483 [2024/02/28 17:31:10.483 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:10.483 [2024/02/28 17:31:10.483 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=9e23dd95e07f0d4c18dc2f7c7c7abc47] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:10.663 [2024/02/28 17:31:10.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626456] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:10.663 [2024/02/28 17:31:10.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626457] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626524214][leader=3]"] 2024-02-29 01:31:10.663 [2024/02/28 17:31:10.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626458] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626528386][leader=3]"] 2024-02-29 01:31:10.663 [2024/02/28 17:31:10.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626459] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:10.663 [2024/02/28 17:31:10.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626460] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626527785][leader=3]"] 2024-02-29 01:31:10.663 [2024/02/28 17:31:10.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626461] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:10.663 [2024/02/28 17:31:10.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626462] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:10.665 [2024/02/28 17:31:10.665 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626523124] [error="node offline[node=8]"] 2024-02-29 01:31:10.665 [2024/02/28 17:31:10.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:10.665 [2024/02/28 17:31:10.665 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:10.665 [2024/02/28 17:31:10.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626547521] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626527784] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:10.666 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:10.667 [2024/02/28 17:31:10.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:10.667 [2024/02/28 17:31:10.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:10.667 [2024/02/28 17:31:10.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:10.667 [2024/02/28 17:31:10.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626463] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761400]"] 2024-02-29 01:31:10.667 [2024/02/28 17:31:10.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626464] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760025]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626465] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760757]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626466] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760430]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626467] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759725]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626468] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759971]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626469] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760744]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626470] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760570]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626471] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761482]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626472] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759628]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626473] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761031]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626474] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760970]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626475] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761308]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626476] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760748]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626477] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760366]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626478] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761143]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626479] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761572]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626480] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760422]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626481] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760192]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626482] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759826]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626483] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761556]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626484] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760696]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626485] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759612]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626486] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759770]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626487] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759849]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626488] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759777]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626489] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760215]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626490] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760199]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626491] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760374]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626492] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759968]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626493] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877626526417]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626494] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742556]"] 2024-02-29 01:31:10.668 [2024/02/28 17:31:10.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626495] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742592]"] 2024-02-29 01:31:10.669 [2024/02/28 17:31:10.669 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626459] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:10.669 [2024/02/28 17:31:10.669 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626459] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:10.669 [2024/02/28 17:31:10.669 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626459] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:10.669 [2024/02/28 17:31:10.669 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626462] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:10.670 [2024/02/28 17:31:10.669 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626462] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:10.670 [2024/02/28 17:31:10.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626462] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:10.670 [2024/02/28 17:31:10.670 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626461] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:10.670 [2024/02/28 17:31:10.670 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626461] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:10.670 [2024/02/28 17:31:10.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626461] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:10.670 [2024/02/28 17:31:10.670 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626456] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:10.670 [2024/02/28 17:31:10.670 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626456] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:10.670 [2024/02/28 17:31:10.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626456] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:10.671 [2024/02/28 17:31:10.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626459] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:10.671 [2024/02/28 17:31:10.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626462] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:10.671 [2024/02/28 17:31:10.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626461] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:10.671 [2024/02/28 17:31:10.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626456] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:10.671 [2024/02/28 17:31:10.671 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=36] [toRemoveNum=4] 2024-02-29 01:31:10.671 [2024/02/28 17:31:10.671 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=41] [waitingTaskNum=0] [segmentTaskNum=41] [channelTaskNum=0] 2024-02-29 01:31:10.673 [2024/02/28 17:31:10.673 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626457] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:10.673 [2024/02/28 17:31:10.673 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626458] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:10.675 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626473] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.675 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626484] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.675 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626495] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.675 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626467] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.675 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626487] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.675 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626483] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.676 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626485] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.676 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626474] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.676 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626464] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.676 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626478] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.677 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626492] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.677 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626468] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.677 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626465] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.677 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626491] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.678 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626476] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.678 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626463] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.678 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626493] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.678 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626490] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.678 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626481] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.678 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626480] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.678 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626470] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.678 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626488] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.679 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626471] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.679 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626469] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.679 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626475] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.679 [2024/02/28 17:31:10.674 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=41] ["time cost"=1.483119ms] 2024-02-29 01:31:10.679 [2024/02/28 17:31:10.679 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=41] [committedNum=36] [toRemoveNum=0] 2024-02-29 01:31:10.679 [2024/02/28 17:31:10.679 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=41] [waitingTaskNum=0] [segmentTaskNum=41] [channelTaskNum=0] 2024-02-29 01:31:10.679 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626486] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.679 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626494] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.679 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626477] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.679 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626460] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:10.679 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626479] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.680 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626482] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.680 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626466] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.680 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626472] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.680 [2024/02/28 17:31:10.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626489] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:10.797 [2024/02/28 17:31:10.796 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=f899adeebef7d703c804920cb8c0cecc] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298936,448039877626299011,448039877626298953,448039877626298931,448039877626298981,448039877626298959,448039877626299033,448039877626299016,448039877626298948,448039877626298976,448039877626299027,448039877626299022,448039877626298965,448039877626299005,448039877626298993,448039877626298987,448039877626298942,448039877626298999,448039877626298970]"] 2024-02-29 01:31:10.797 [2024/02/28 17:31:10.796 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=f899adeebef7d703c804920cb8c0cecc] [collectionID=448039877626298924] 2024-02-29 01:31:10.797 [2024/02/28 17:31:10.796 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=f899adeebef7d703c804920cb8c0cecc] [collectionID=448039877626298924] 2024-02-29 01:31:10.797 [2024/02/28 17:31:10.796 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:10.797 [2024/02/28 17:31:10.797 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=f899adeebef7d703c804920cb8c0cecc] [collectionID=448039877626298924] 2024-02-29 01:31:10.797 [2024/02/28 17:31:10.797 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=f899adeebef7d703c804920cb8c0cecc] [collectionID=448039877626298924] 2024-02-29 01:31:10.802 [2024/02/28 17:31:10.802 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=c99435d03905630f166b8e414ec1cb86] [collections="[448039877626298924]"] 2024-02-29 01:31:11.167 [2024/02/28 17:31:11.167 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=41] [waitingTaskNum=0] [segmentTaskNum=41] [channelTaskNum=0] 2024-02-29 01:31:11.168 [2024/02/28 17:31:11.168 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=41] ["time cost"=186.97µs] 2024-02-29 01:31:11.168 [2024/02/28 17:31:11.168 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=41] [committedNum=0] [toRemoveNum=0] 2024-02-29 01:31:11.168 [2024/02/28 17:31:11.168 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=41] [waitingTaskNum=0] [segmentTaskNum=41] [channelTaskNum=0] 2024-02-29 01:31:11.206 [2024/02/28 17:31:11.206 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626475] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.235 [2024/02/28 17:31:11.235 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626492] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.283 [2024/02/28 17:31:11.283 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626484] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.311 [2024/02/28 17:31:11.311 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626483] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.319 [2024/02/28 17:31:11.319 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626466] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.351 [2024/02/28 17:31:11.351 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626464] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.361 [2024/02/28 17:31:11.361 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=74bd7fe6868dc273ef2fcfc9c12c60af] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298948,448039877626298931,448039877626298970,448039877626298936,448039877626298999,448039877626299016,448039877626299033,448039877626299027,448039877626298987,448039877626298942,448039877626298976,448039877626299005,448039877626298981,448039877626299011,448039877626298959,448039877626298953,448039877626299022,448039877626298993,448039877626298965]"] 2024-02-29 01:31:11.361 [2024/02/28 17:31:11.361 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=74bd7fe6868dc273ef2fcfc9c12c60af] [collectionID=448039877626298924] 2024-02-29 01:31:11.361 [2024/02/28 17:31:11.361 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=74bd7fe6868dc273ef2fcfc9c12c60af] [collectionID=448039877626298924] 2024-02-29 01:31:11.361 [2024/02/28 17:31:11.361 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:11.362 [2024/02/28 17:31:11.362 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=74bd7fe6868dc273ef2fcfc9c12c60af] [collectionID=448039877626298924] 2024-02-29 01:31:11.362 [2024/02/28 17:31:11.362 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=74bd7fe6868dc273ef2fcfc9c12c60af] [collectionID=448039877626298924] 2024-02-29 01:31:11.367 [2024/02/28 17:31:11.367 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=21545b5da93fb11b79ab8c79f1fd6381] [collections="[448039877626298924]"] 2024-02-29 01:31:11.374 [2024/02/28 17:31:11.374 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626463] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.375 [2024/02/28 17:31:11.374 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:11.375 [2024/02/28 17:31:11.375 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=6bd512eeff36a3105adf3bf41b4f3c18] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:11.397 [2024/02/28 17:31:11.394 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626481] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.397 [2024/02/28 17:31:11.395 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626485] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.397 [2024/02/28 17:31:11.397 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626479] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.436 [2024/02/28 17:31:11.436 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626489] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.436 [2024/02/28 17:31:11.436 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626472] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.436 [2024/02/28 17:31:11.436 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626488] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.436 [2024/02/28 17:31:11.436 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626458] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:11.441 [2024/02/28 17:31:11.441 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626458] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] [elapsed=5.140801ms] 2024-02-29 01:31:11.448 [2024/02/28 17:31:11.447 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626473] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.448 [2024/02/28 17:31:11.448 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626494] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.487 [2024/02/28 17:31:11.487 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626476] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.489 [2024/02/28 17:31:11.489 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626491] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.489 [2024/02/28 17:31:11.489 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626469] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.491 [2024/02/28 17:31:11.491 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626487] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.491 [2024/02/28 17:31:11.491 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626482] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.492 [2024/02/28 17:31:11.492 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626480] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.495 [2024/02/28 17:31:11.495 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626467] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.495 [2024/02/28 17:31:11.495 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626495] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.507 [2024/02/28 17:31:11.507 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626475] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:11.507 [2024/02/28 17:31:11.507 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626475] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:11.513 [2024/02/28 17:31:11.513 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626490] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.513 [2024/02/28 17:31:11.513 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626493] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626526417] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.513 [2024/02/28 17:31:11.513 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626486] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.524 [2024/02/28 17:31:11.524 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626460] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:11.524 [2024/02/28 17:31:11.524 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626468] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.527 [2024/02/28 17:31:11.527 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626460] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] [elapsed=2.684411ms] 2024-02-29 01:31:11.555 [2024/02/28 17:31:11.555 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626457] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:11.557 [2024/02/28 17:31:11.557 +00:00] [INFO] [task/executor.go:211] ["load segments done"] [taskID=1709136625261] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527784] [node=3] [source=segment_checker] [shardLeader=3] [elapsed=28.753974288s] 2024-02-29 01:31:11.563 [2024/02/28 17:31:11.563 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626477] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.581 [2024/02/28 17:31:11.580 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626470] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.581 [2024/02/28 17:31:11.580 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626478] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.585 [2024/02/28 17:31:11.584 +00:00] [WARN] [task/executor.go:468] ["failed to sync distribution"] [taskID=1709136626457] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:11.585 [2024/02/28 17:31:11.584 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626457] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:11.594 [2024/02/28 17:31:11.593 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626474] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.617 [2024/02/28 17:31:11.617 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626471] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.640 [2024/02/28 17:31:11.639 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626465] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:11.663 [2024/02/28 17:31:11.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626496] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:11.663 [2024/02/28 17:31:11.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626497] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:11.663 [2024/02/28 17:31:11.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626498] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:11.663 [2024/02/28 17:31:11.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626499] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626547521] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.665 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.666 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.666 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.666 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.666 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.666 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.666 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:11.666 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626527784] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:11.666 [2024/02/28 17:31:11.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:11.666 [2024/02/28 17:31:11.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:11.666 [2024/02/28 17:31:11.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:11.666 [2024/02/28 17:31:11.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626497] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626497] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626497] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626496] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626496] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626496] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626499] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626499] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626499] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626498] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626498] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626498] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626497] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626496] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626499] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626498] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=0] [toRemoveNum=4] 2024-02-29 01:31:11.667 [2024/02/28 17:31:11.667 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=41] [waitingTaskNum=0] [segmentTaskNum=41] [channelTaskNum=0] 2024-02-29 01:31:11.668 [2024/02/28 17:31:11.668 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=36] ["time cost"=159.832µs] 2024-02-29 01:31:11.668 [2024/02/28 17:31:11.668 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626457] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626524214] 2024-02-29 01:31:11.668 [2024/02/28 17:31:11.668 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626458] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626528386] 2024-02-29 01:31:11.668 [2024/02/28 17:31:11.668 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626460] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626527785] 2024-02-29 01:31:11.668 [2024/02/28 17:31:11.668 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626475] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:11.668 [2024/02/28 17:31:11.668 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:11.669 [2024/02/28 17:31:11.668 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626475] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761308] 2024-02-29 01:31:11.669 [2024/02/28 17:31:11.668 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136625261] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626527784] 2024-02-29 01:31:11.669 [2024/02/28 17:31:11.669 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=36] [committedNum=0] [toRemoveNum=5] 2024-02-29 01:31:11.669 [2024/02/28 17:31:11.669 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=36] [waitingTaskNum=0] [segmentTaskNum=36] [channelTaskNum=0] 2024-02-29 01:31:11.727 [2024/02/28 17:31:11.727 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626492] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:11.727 [2024/02/28 17:31:11.727 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626492] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:11.787 [2024/02/28 17:31:11.787 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=37e5fcf177b7fe5100878eacbf2ab324] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298931,448039877626299022,448039877626298953,448039877626298987,448039877626299027,448039877626298942,448039877626298993,448039877626299033,448039877626299005,448039877626298936,448039877626298948,448039877626298999,448039877626299016,448039877626298981,448039877626298970,448039877626298965,448039877626298959,448039877626299011,448039877626298976]"] 2024-02-29 01:31:11.787 [2024/02/28 17:31:11.787 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=37e5fcf177b7fe5100878eacbf2ab324] [collectionID=448039877626298924] 2024-02-29 01:31:11.787 [2024/02/28 17:31:11.787 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=37e5fcf177b7fe5100878eacbf2ab324] [collectionID=448039877626298924] 2024-02-29 01:31:11.787 [2024/02/28 17:31:11.787 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:11.789 [2024/02/28 17:31:11.788 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=37e5fcf177b7fe5100878eacbf2ab324] [collectionID=448039877626298924] 2024-02-29 01:31:11.789 [2024/02/28 17:31:11.789 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=37e5fcf177b7fe5100878eacbf2ab324] [collectionID=448039877626298924] 2024-02-29 01:31:11.796 [2024/02/28 17:31:11.795 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=59a4a0a779e6c5eba03a270569fcbd85] [collections="[448039877626298924]"] 2024-02-29 01:31:11.961 [2024/02/28 17:31:11.961 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626466] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:11.961 [2024/02/28 17:31:11.961 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626466] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:12.042 [2024/02/28 17:31:12.042 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=3b23c60de224f8349f8c737836689d0f] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298948,448039877626298999,448039877626298953,448039877626299011,448039877626299005,448039877626298976,448039877626299016,448039877626298931,448039877626298970,448039877626298965,448039877626298993,448039877626299027,448039877626299033,448039877626298942,448039877626298981,448039877626299022,448039877626298936,448039877626298959,448039877626298987]"] 2024-02-29 01:31:12.042 [2024/02/28 17:31:12.042 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=3b23c60de224f8349f8c737836689d0f] [collectionID=448039877626298924] 2024-02-29 01:31:12.042 [2024/02/28 17:31:12.042 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=3b23c60de224f8349f8c737836689d0f] [collectionID=448039877626298924] 2024-02-29 01:31:12.043 [2024/02/28 17:31:12.042 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:12.043 [2024/02/28 17:31:12.043 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=3b23c60de224f8349f8c737836689d0f] [collectionID=448039877626298924] 2024-02-29 01:31:12.043 [2024/02/28 17:31:12.043 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=3b23c60de224f8349f8c737836689d0f] [collectionID=448039877626298924] 2024-02-29 01:31:12.051 [2024/02/28 17:31:12.051 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=1cbf1e67b64a2f77b3f1925ad249a82c] [collections="[448039877626298924]"] 2024-02-29 01:31:12.156 [2024/02/28 17:31:12.155 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:12.156 [2024/02/28 17:31:12.155 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=37145312b2dee357d40ed637de479fb0] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:12.168 [2024/02/28 17:31:12.168 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=36] [waitingTaskNum=0] [segmentTaskNum=36] [channelTaskNum=0] 2024-02-29 01:31:12.169 [2024/02/28 17:31:12.169 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=34] ["time cost"=200.255µs] 2024-02-29 01:31:12.169 [2024/02/28 17:31:12.169 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626466] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:12.169 [2024/02/28 17:31:12.169 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:12.169 [2024/02/28 17:31:12.169 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626466] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760430] 2024-02-29 01:31:12.169 [2024/02/28 17:31:12.169 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626492] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:12.169 [2024/02/28 17:31:12.169 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:12.169 [2024/02/28 17:31:12.169 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626492] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759968] 2024-02-29 01:31:12.169 [2024/02/28 17:31:12.169 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=34] [committedNum=0] [toRemoveNum=2] 2024-02-29 01:31:12.169 [2024/02/28 17:31:12.169 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=34] [waitingTaskNum=0] [segmentTaskNum=34] [channelTaskNum=0] 2024-02-29 01:31:12.169 [2024/02/28 17:31:12.169 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:12.169 [2024/02/28 17:31:12.169 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=9a4893638d2c196a94779cac78ee93df] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:12.420 [2024/02/28 17:31:12.420 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=3ab92620b062075a1adb364c248ae9ea] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298970,448039877626298965,448039877626299033,448039877626298931,448039877626299027,448039877626299016,448039877626298987,448039877626298993,448039877626298942,448039877626298959,448039877626298953,448039877626298936,448039877626298976,448039877626299022,448039877626298999,448039877626298981,448039877626298948,448039877626299011,448039877626299005]"] 2024-02-29 01:31:12.420 [2024/02/28 17:31:12.420 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=3ab92620b062075a1adb364c248ae9ea] [collectionID=448039877626298924] 2024-02-29 01:31:12.420 [2024/02/28 17:31:12.420 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=3ab92620b062075a1adb364c248ae9ea] [collectionID=448039877626298924] 2024-02-29 01:31:12.420 [2024/02/28 17:31:12.420 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:12.421 [2024/02/28 17:31:12.421 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=3ab92620b062075a1adb364c248ae9ea] [collectionID=448039877626298924] 2024-02-29 01:31:12.421 [2024/02/28 17:31:12.421 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=3ab92620b062075a1adb364c248ae9ea] [collectionID=448039877626298924] 2024-02-29 01:31:12.426 [2024/02/28 17:31:12.426 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=f58a052999f06635d1311060bc8b8c00] [collections="[448039877626298924]"] 2024-02-29 01:31:12.431 [2024/02/28 17:31:12.431 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:12.431 [2024/02/28 17:31:12.431 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=ab350cf6404f6b3384e1043405d4dd5e] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:12.435 [2024/02/28 17:31:12.434 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626527182] 2024-02-29 01:31:12.435 [2024/02/28 17:31:12.435 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=92081e8e27f68d520c41813221fe76a1] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626527182]"] 2024-02-29 01:31:12.464 [2024/02/28 17:31:12.464 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626483] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:12.464 [2024/02/28 17:31:12.464 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626483] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:12.594 [2024/02/28 17:31:12.590 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:12.594 [2024/02/28 17:31:12.590 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=9ca71f782d12c42622198cd753b652e9] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:12.643 [2024/02/28 17:31:12.643 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626484] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:12.643 [2024/02/28 17:31:12.643 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626484] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:12.664 [2024/02/28 17:31:12.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626500] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626524214][leader=3]"] 2024-02-29 01:31:12.664 [2024/02/28 17:31:12.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626501] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:12.664 [2024/02/28 17:31:12.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626502] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:12.664 [2024/02/28 17:31:12.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626503] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626527785][leader=3]"] 2024-02-29 01:31:12.664 [2024/02/28 17:31:12.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626504] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:12.664 [2024/02/28 17:31:12.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626505] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626528386][leader=3]"] 2024-02-29 01:31:12.664 [2024/02/28 17:31:12.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626506] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:12.664 [2024/02/28 17:31:12.664 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626523124] [error="node offline[node=8]"] 2024-02-29 01:31:12.664 [2024/02/28 17:31:12.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:12.664 [2024/02/28 17:31:12.664 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:12.664 [2024/02/28 17:31:12.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626547521] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:12.665 [2024/02/28 17:31:12.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:12.666 [2024/02/28 17:31:12.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626507] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760430]"] 2024-02-29 01:31:12.666 [2024/02/28 17:31:12.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626508] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761308]"] 2024-02-29 01:31:12.666 [2024/02/28 17:31:12.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626509] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759968]"] 2024-02-29 01:31:12.686 [2024/02/28 17:31:12.676 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626501] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:12.686 [2024/02/28 17:31:12.677 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626501] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:12.686 [2024/02/28 17:31:12.677 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626501] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:12.686 [2024/02/28 17:31:12.678 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626504] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:12.686 [2024/02/28 17:31:12.678 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626504] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:12.686 [2024/02/28 17:31:12.678 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626504] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:12.699 [2024/02/28 17:31:12.699 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626506] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:12.699 [2024/02/28 17:31:12.699 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:12.699 [2024/02/28 17:31:12.699 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=55d761a8174ce66c918db6d625ed13c5] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:12.699 [2024/02/28 17:31:12.699 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626506] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:12.699 [2024/02/28 17:31:12.699 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626506] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:12.700 [2024/02/28 17:31:12.700 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626502] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:12.700 [2024/02/28 17:31:12.700 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626502] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:12.700 [2024/02/28 17:31:12.700 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626502] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:12.700 [2024/02/28 17:31:12.700 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626501] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:12.700 [2024/02/28 17:31:12.700 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626504] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:12.700 [2024/02/28 17:31:12.700 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626506] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:12.700 [2024/02/28 17:31:12.700 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626502] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:12.700 [2024/02/28 17:31:12.700 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=6] [toRemoveNum=4] 2024-02-29 01:31:12.700 [2024/02/28 17:31:12.700 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=40] [waitingTaskNum=0] [segmentTaskNum=40] [channelTaskNum=0] 2024-02-29 01:31:12.701 [2024/02/28 17:31:12.701 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626503] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:12.701 [2024/02/28 17:31:12.701 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626508] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:12.701 [2024/02/28 17:31:12.701 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626505] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:12.701 [2024/02/28 17:31:12.701 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626507] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:12.701 [2024/02/28 17:31:12.701 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626509] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:12.701 [2024/02/28 17:31:12.701 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=38] ["time cost"=307.364µs] 2024-02-29 01:31:12.701 [2024/02/28 17:31:12.701 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626500] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:12.701 [2024/02/28 17:31:12.701 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626484] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:12.702 [2024/02/28 17:31:12.701 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:12.702 [2024/02/28 17:31:12.702 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626484] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760696] 2024-02-29 01:31:12.702 [2024/02/28 17:31:12.702 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626483] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:12.702 [2024/02/28 17:31:12.702 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:12.702 [2024/02/28 17:31:12.702 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626483] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761556] 2024-02-29 01:31:12.702 [2024/02/28 17:31:12.702 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=38] [committedNum=6] [toRemoveNum=2] 2024-02-29 01:31:12.702 [2024/02/28 17:31:12.702 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=38] [waitingTaskNum=0] [segmentTaskNum=38] [channelTaskNum=0] 2024-02-29 01:31:12.704 [2024/02/28 17:31:12.704 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:12.704 [2024/02/28 17:31:12.704 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=833a7e4e0384b0b706ad58f61fc49562] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626485] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626464] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626469] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626481] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626469] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626479] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626472] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626479] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626472] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626485] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626494] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626481] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626464] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626494] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626463] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626463] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626488] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626488] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626487] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626487] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626476] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626476] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626482] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626491] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626495] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626482] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626491] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626495] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626489] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:12.804 [2024/02/28 17:31:12.804 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626489] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626467] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626473] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626473] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626486] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626467] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626486] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626480] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626478] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626468] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626478] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626480] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626470] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626490] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626470] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626490] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626477] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626477] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:12.856 [2024/02/28 17:31:12.856 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626468] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:12.857 [2024/02/28 17:31:12.857 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626474] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:12.857 [2024/02/28 17:31:12.857 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626474] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:12.886 [2024/02/28 17:31:12.885 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626471] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:12.886 [2024/02/28 17:31:12.885 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626471] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:12.922 [2024/02/28 17:31:12.922 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626465] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:12.922 [2024/02/28 17:31:12.922 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626465] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:13.107 [2024/02/28 17:31:13.107 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626507] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:13.107 [2024/02/28 17:31:13.107 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626509] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:13.114 [2024/02/28 17:31:13.114 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626503] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:13.114 [2024/02/28 17:31:13.114 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626505] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:13.114 [2024/02/28 17:31:13.114 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626508] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:13.120 [2024/02/28 17:31:13.120 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626505] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] [elapsed=5.731119ms] 2024-02-29 01:31:13.121 [2024/02/28 17:31:13.121 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626503] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] [elapsed=6.629415ms] 2024-02-29 01:31:13.170 [2024/02/28 17:31:13.170 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=38] [waitingTaskNum=0] [segmentTaskNum=38] [channelTaskNum=0] 2024-02-29 01:31:13.170 [2024/02/28 17:31:13.170 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=9] ["time cost"=84.092µs] 2024-02-29 01:31:13.170 [2024/02/28 17:31:13.170 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626503] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626527785] 2024-02-29 01:31:13.170 [2024/02/28 17:31:13.170 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626505] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626528386] 2024-02-29 01:31:13.170 [2024/02/28 17:31:13.170 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626486] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:13.170 [2024/02/28 17:31:13.170 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:13.170 [2024/02/28 17:31:13.170 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626486] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759770] 2024-02-29 01:31:13.170 [2024/02/28 17:31:13.170 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626477] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:13.170 [2024/02/28 17:31:13.170 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:13.170 [2024/02/28 17:31:13.170 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626477] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760366] 2024-02-29 01:31:13.170 [2024/02/28 17:31:13.170 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626479] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:13.171 [2024/02/28 17:31:13.170 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:13.171 [2024/02/28 17:31:13.170 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626479] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761572] 2024-02-29 01:31:13.171 [2024/02/28 17:31:13.170 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626489] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:13.171 [2024/02/28 17:31:13.170 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:13.171 [2024/02/28 17:31:13.171 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626489] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760215] 2024-02-29 01:31:13.171 [2024/02/28 17:31:13.171 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626472] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:13.171 [2024/02/28 17:31:13.171 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:13.171 [2024/02/28 17:31:13.171 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626472] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759628] 2024-02-29 01:31:13.171 [2024/02/28 17:31:13.171 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626494] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:13.171 [2024/02/28 17:31:13.171 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:13.171 [2024/02/28 17:31:13.171 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626494] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742556] 2024-02-29 01:31:13.171 [2024/02/28 17:31:13.171 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626482] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:13.171 [2024/02/28 17:31:13.171 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:13.171 [2024/02/28 17:31:13.171 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626482] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759826] 2024-02-29 01:31:13.172 [2024/02/28 17:31:13.171 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626480] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626480] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760422] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626470] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626470] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760570] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626473] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626473] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761031] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626495] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626495] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742592] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626467] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626467] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759725] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626487] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626487] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759849] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626485] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626485] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759612] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626474] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626474] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760970] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626464] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626464] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760025] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626478] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.171 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626478] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761143] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626468] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626468] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759971] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626465] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626465] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760757] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626491] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626491] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760374] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626476] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626476] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760748] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626463] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626463] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761400] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626490] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626490] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760199] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626481] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626481] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760192] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626488] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626488] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759777] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626471] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626471] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761482] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626469] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626469] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760744] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=9] [committedNum=0] [toRemoveNum=29] 2024-02-29 01:31:13.173 [2024/02/28 17:31:13.172 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=9] [waitingTaskNum=0] [segmentTaskNum=9] [channelTaskNum=0] 2024-02-29 01:31:13.237 [2024/02/28 17:31:13.237 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626500] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:13.290 [2024/02/28 17:31:13.289 +00:00] [WARN] [task/executor.go:468] ["failed to sync distribution"] [taskID=1709136626500] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:13.290 [2024/02/28 17:31:13.289 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626500] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:13.322 [2024/02/28 17:31:13.321 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=8633d4da1679af992f15a5e77c80aa63] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626299022,448039877626298936,448039877626298993,448039877626298965,448039877626298942,448039877626298931,448039877626298999,448039877626299033,448039877626299011,448039877626299005,448039877626298948,448039877626298970,448039877626298959,448039877626298987,448039877626298981,448039877626298953,448039877626299027,448039877626299016,448039877626298976]"] 2024-02-29 01:31:13.322 [2024/02/28 17:31:13.322 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=8633d4da1679af992f15a5e77c80aa63] [collectionID=448039877626298924] 2024-02-29 01:31:13.322 [2024/02/28 17:31:13.322 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=8633d4da1679af992f15a5e77c80aa63] [collectionID=448039877626298924] 2024-02-29 01:31:13.322 [2024/02/28 17:31:13.322 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:13.323 [2024/02/28 17:31:13.323 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=8633d4da1679af992f15a5e77c80aa63] [collectionID=448039877626298924] 2024-02-29 01:31:13.323 [2024/02/28 17:31:13.323 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=8633d4da1679af992f15a5e77c80aa63] [collectionID=448039877626298924] 2024-02-29 01:31:13.328 [2024/02/28 17:31:13.328 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=c3aae270831f9db16d32f11eea19ee7a] [collections="[448039877626298924]"] 2024-02-29 01:31:13.332 [2024/02/28 17:31:13.332 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626507] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:13.332 [2024/02/28 17:31:13.332 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626507] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:13.362 [2024/02/28 17:31:13.361 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626509] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:13.362 [2024/02/28 17:31:13.362 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626509] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:13.388 [2024/02/28 17:31:13.388 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626508] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:13.388 [2024/02/28 17:31:13.388 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626508] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:13.438 [2024/02/28 17:31:13.438 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:13.438 [2024/02/28 17:31:13.438 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=a14121a46c85dbcfaf596c8630a854f2] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:13.554 [2024/02/28 17:31:13.554 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:13.554 [2024/02/28 17:31:13.554 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=38d5a57761091882d42b00830c5fe04d] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:13.664 [2024/02/28 17:31:13.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626510] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:13.664 [2024/02/28 17:31:13.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626511] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:13.664 [2024/02/28 17:31:13.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626512] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626527785][leader=3]"] 2024-02-29 01:31:13.664 [2024/02/28 17:31:13.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626513] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:13.664 [2024/02/28 17:31:13.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626514] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:13.664 [2024/02/28 17:31:13.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626515] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626528386][leader=3]"] 2024-02-29 01:31:13.664 [2024/02/28 17:31:13.664 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626523124] [error="node offline[node=8]"] 2024-02-29 01:31:13.664 [2024/02/28 17:31:13.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:13.664 [2024/02/28 17:31:13.664 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:13.664 [2024/02/28 17:31:13.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626547521] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.665 [2024/02/28 17:31:13.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626516] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742556]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626517] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742592]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626518] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761400]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626519] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760025]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626520] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760757]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626521] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759725]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626522] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759971]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626523] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760744]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626524] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760570]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626525] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761482]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626526] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759628]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626527] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761031]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626528] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760970]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626529] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760366]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626530] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760748]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626531] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761143]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626532] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760192]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626533] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760422]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626534] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761572]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626535] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760696]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626536] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759826]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626537] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761556]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626538] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759770]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626539] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759612]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626540] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759849]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626541] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759777]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626542] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760199]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626543] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760215]"] 2024-02-29 01:31:13.666 [2024/02/28 17:31:13.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626544] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760374]"] 2024-02-29 01:31:13.673 [2024/02/28 17:31:13.671 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626513] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:13.674 [2024/02/28 17:31:13.671 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626513] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:13.674 [2024/02/28 17:31:13.671 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626513] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:13.674 [2024/02/28 17:31:13.672 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626511] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:13.674 [2024/02/28 17:31:13.672 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626511] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:13.674 [2024/02/28 17:31:13.672 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626511] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:13.674 [2024/02/28 17:31:13.672 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626510] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:13.674 [2024/02/28 17:31:13.672 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626510] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:13.674 [2024/02/28 17:31:13.672 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626510] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:13.674 [2024/02/28 17:31:13.672 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626514] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:13.674 [2024/02/28 17:31:13.672 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626514] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:13.674 [2024/02/28 17:31:13.672 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626514] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:13.675 [2024/02/28 17:31:13.674 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626513] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:13.675 [2024/02/28 17:31:13.674 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626511] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:13.675 [2024/02/28 17:31:13.675 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626510] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:13.675 [2024/02/28 17:31:13.675 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626514] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:13.675 [2024/02/28 17:31:13.675 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=31] [toRemoveNum=4] 2024-02-29 01:31:13.675 [2024/02/28 17:31:13.675 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=40] [waitingTaskNum=0] [segmentTaskNum=40] [channelTaskNum=0] 2024-02-29 01:31:13.687 [2024/02/28 17:31:13.685 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=36] ["time cost"=1.277993ms] 2024-02-29 01:31:13.687 [2024/02/28 17:31:13.685 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626500] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626524214] 2024-02-29 01:31:13.687 [2024/02/28 17:31:13.685 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626508] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:13.687 [2024/02/28 17:31:13.685 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:13.687 [2024/02/28 17:31:13.685 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626508] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761308] 2024-02-29 01:31:13.687 [2024/02/28 17:31:13.686 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626507] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:13.687 [2024/02/28 17:31:13.686 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:13.687 [2024/02/28 17:31:13.686 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626507] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760430] 2024-02-29 01:31:13.687 [2024/02/28 17:31:13.686 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626509] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:13.687 [2024/02/28 17:31:13.686 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:13.687 [2024/02/28 17:31:13.686 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626509] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759968] 2024-02-29 01:31:13.687 [2024/02/28 17:31:13.686 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=36] [committedNum=31] [toRemoveNum=4] 2024-02-29 01:31:13.687 [2024/02/28 17:31:13.686 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=36] [waitingTaskNum=0] [segmentTaskNum=36] [channelTaskNum=0] 2024-02-29 01:31:13.687 [2024/02/28 17:31:13.686 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626543] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.687 [2024/02/28 17:31:13.686 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626512] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:13.687 [2024/02/28 17:31:13.687 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626515] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.687 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626523] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.687 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626530] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.687 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626540] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.687 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626538] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.688 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626518] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.688 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626536] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.688 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626537] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.688 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626524] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.688 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626541] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.688 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626529] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.688 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626528] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.688 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626519] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.688 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626521] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.689 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626516] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.689 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626526] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.688 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626544] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.689 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626527] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.689 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626539] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.689 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626520] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.689 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626542] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626533] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626531] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626534] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626517] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626535] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626532] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626525] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.693 [2024/02/28 17:31:13.691 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626522] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:13.813 [2024/02/28 17:31:13.813 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:13.813 [2024/02/28 17:31:13.813 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=f077cca451131f87398c6cbe45fefe65] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:13.817 [2024/02/28 17:31:13.817 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:13.817 [2024/02/28 17:31:13.817 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=ec7aa47564a8b7ff1696202986b7f0e6] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:13.911 [2024/02/28 17:31:13.911 +00:00] [DEBUG] [config/etcd_source.go:143] ["etcd refreshConfigurations"] [prefix=by-dev/config] [endpoints="[inverted-corn-136000-8-75-4542-etcd:2379]"] 2024-02-29 01:31:14.147 [2024/02/28 17:31:14.146 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626532] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.147 [2024/02/28 17:31:14.147 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626519] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.167 [2024/02/28 17:31:14.167 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=36] [waitingTaskNum=0] [segmentTaskNum=36] [channelTaskNum=0] 2024-02-29 01:31:14.168 [2024/02/28 17:31:14.168 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=36] ["time cost"=192.246µs] 2024-02-29 01:31:14.168 [2024/02/28 17:31:14.168 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=36] [committedNum=0] [toRemoveNum=0] 2024-02-29 01:31:14.168 [2024/02/28 17:31:14.168 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=36] [waitingTaskNum=0] [segmentTaskNum=36] [channelTaskNum=0] 2024-02-29 01:31:14.217 [2024/02/28 17:31:14.217 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626522] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.217 [2024/02/28 17:31:14.217 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626543] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.262 [2024/02/28 17:31:14.262 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626542] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.262 [2024/02/28 17:31:14.262 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626533] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.262 [2024/02/28 17:31:14.262 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626523] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.262 [2024/02/28 17:31:14.262 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626540] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.262 [2024/02/28 17:31:14.262 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626538] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.262 [2024/02/28 17:31:14.262 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626530] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.284 [2024/02/28 17:31:14.282 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626544] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.285 [2024/02/28 17:31:14.285 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626518] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.289 [2024/02/28 17:31:14.287 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626535] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.290 [2024/02/28 17:31:14.287 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626516] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.290 [2024/02/28 17:31:14.288 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626534] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.290 [2024/02/28 17:31:14.288 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626526] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.290 [2024/02/28 17:31:14.289 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626541] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.297 [2024/02/28 17:31:14.296 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626539] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.297 [2024/02/28 17:31:14.296 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626512] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:14.297 [2024/02/28 17:31:14.296 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626521] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.297 [2024/02/28 17:31:14.296 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626517] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.297 [2024/02/28 17:31:14.297 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626537] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.300 [2024/02/28 17:31:14.300 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626512] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] [elapsed=3.610027ms] 2024-02-29 01:31:14.314 [2024/02/28 17:31:14.314 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626525] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.314 [2024/02/28 17:31:14.314 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626529] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.314 [2024/02/28 17:31:14.314 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626536] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.338 [2024/02/28 17:31:14.338 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626527] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.338 [2024/02/28 17:31:14.338 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626531] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.338 [2024/02/28 17:31:14.338 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626524] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.338 [2024/02/28 17:31:14.338 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626520] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.338 [2024/02/28 17:31:14.338 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626528] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.339 [2024/02/28 17:31:14.339 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=a47e172aedb7892e9e9ee14b6ae1a375] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298931,448039877626298959,448039877626299033,448039877626299005,448039877626299016,448039877626299011,448039877626298970,448039877626298953,448039877626298948,448039877626299027,448039877626298999,448039877626298993,448039877626298987,448039877626298942,448039877626298981,448039877626298936,448039877626298976,448039877626299022,448039877626298965]"] 2024-02-29 01:31:14.339 [2024/02/28 17:31:14.339 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=a47e172aedb7892e9e9ee14b6ae1a375] [collectionID=448039877626298924] 2024-02-29 01:31:14.339 [2024/02/28 17:31:14.339 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=a47e172aedb7892e9e9ee14b6ae1a375] [collectionID=448039877626298924] 2024-02-29 01:31:14.339 [2024/02/28 17:31:14.339 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:14.340 [2024/02/28 17:31:14.340 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=a47e172aedb7892e9e9ee14b6ae1a375] [collectionID=448039877626298924] 2024-02-29 01:31:14.340 [2024/02/28 17:31:14.340 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=a47e172aedb7892e9e9ee14b6ae1a375] [collectionID=448039877626298924] 2024-02-29 01:31:14.366 [2024/02/28 17:31:14.366 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626515] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:14.370 [2024/02/28 17:31:14.370 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626515] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] [elapsed=3.802412ms] 2024-02-29 01:31:14.383 [2024/02/28 17:31:14.382 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=8114162680becb0a95277db87ca06acd] [collections="[448039877626298924]"] 2024-02-29 01:31:14.663 [2024/02/28 17:31:14.663 +00:00] [DEBUG] [checkers/leader_checker.go:138] ["leader checker append a segment to set"] [collectionID=448039877626298924] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [leaderViewID=3] [segmentID=448039877626543860] [nodeID=3] 2024-02-29 01:31:14.664 [2024/02/28 17:31:14.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626545] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:14.664 [2024/02/28 17:31:14.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626546] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626524214][leader=3]"] 2024-02-29 01:31:14.664 [2024/02/28 17:31:14.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626547] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:14.664 [2024/02/28 17:31:14.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626548] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:14.664 [2024/02/28 17:31:14.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626549] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626523124] [error="node offline[node=8]"] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626547521] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.665 [2024/02/28 17:31:14.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:14.666 [2024/02/28 17:31:14.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626550] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760430]"] 2024-02-29 01:31:14.666 [2024/02/28 17:31:14.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626551] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761308]"] 2024-02-29 01:31:14.666 [2024/02/28 17:31:14.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626552] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759968]"] 2024-02-29 01:31:14.685 [2024/02/28 17:31:14.684 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626548] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.684 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626548] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.685 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626548] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.685 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626549] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.685 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626549] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.685 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626549] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.685 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626547] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.685 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626547] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.685 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626547] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.685 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626545] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.686 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626545] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.686 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626545] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.686 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626548] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.686 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626549] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.686 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626547] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.686 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626545] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.686 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=4] [toRemoveNum=4] 2024-02-29 01:31:14.688 [2024/02/28 17:31:14.686 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=40] [waitingTaskNum=0] [segmentTaskNum=40] [channelTaskNum=0] 2024-02-29 01:31:14.690 [2024/02/28 17:31:14.689 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626546] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:14.691 [2024/02/28 17:31:14.689 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=38] ["time cost"=924.181µs] 2024-02-29 01:31:14.691 [2024/02/28 17:31:14.690 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626512] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626527785] 2024-02-29 01:31:14.691 [2024/02/28 17:31:14.690 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626515] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626528386] 2024-02-29 01:31:14.691 [2024/02/28 17:31:14.690 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=38] [committedNum=4] [toRemoveNum=2] 2024-02-29 01:31:14.691 [2024/02/28 17:31:14.690 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=38] [waitingTaskNum=0] [segmentTaskNum=38] [channelTaskNum=0] 2024-02-29 01:31:14.691 [2024/02/28 17:31:14.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626550] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:14.691 [2024/02/28 17:31:14.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626552] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:14.691 [2024/02/28 17:31:14.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626551] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:14.817 [2024/02/28 17:31:14.817 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626551] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.817 [2024/02/28 17:31:14.817 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626550] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.839 [2024/02/28 17:31:14.839 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626552] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:14.856 [2024/02/28 17:31:14.855 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626546] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:14.964 [2024/02/28 17:31:14.963 +00:00] [WARN] [task/executor.go:468] ["failed to sync distribution"] [taskID=1709136626546] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:14.964 [2024/02/28 17:31:14.963 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626546] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:15.167 [2024/02/28 17:31:15.167 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=38] [waitingTaskNum=0] [segmentTaskNum=38] [channelTaskNum=0] 2024-02-29 01:31:15.168 [2024/02/28 17:31:15.168 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=37] ["time cost"=263.996µs] 2024-02-29 01:31:15.168 [2024/02/28 17:31:15.168 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626546] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626524214] 2024-02-29 01:31:15.168 [2024/02/28 17:31:15.168 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=37] [committedNum=0] [toRemoveNum=1] 2024-02-29 01:31:15.168 [2024/02/28 17:31:15.168 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=37] [waitingTaskNum=0] [segmentTaskNum=37] [channelTaskNum=0] 2024-02-29 01:31:15.280 [2024/02/28 17:31:15.280 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626519] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:15.280 [2024/02/28 17:31:15.280 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626519] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:15.590 [2024/02/28 17:31:15.590 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626543] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:15.590 [2024/02/28 17:31:15.590 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626543] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:15.590 [2024/02/28 17:31:15.590 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626518] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:15.590 [2024/02/28 17:31:15.590 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626518] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:15.590 [2024/02/28 17:31:15.590 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626533] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:15.590 [2024/02/28 17:31:15.590 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626533] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:15.590 [2024/02/28 17:31:15.590 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626522] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:15.590 [2024/02/28 17:31:15.590 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626542] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:15.590 [2024/02/28 17:31:15.590 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626522] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:15.591 [2024/02/28 17:31:15.590 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626542] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:15.591 [2024/02/28 17:31:15.590 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626526] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:15.591 [2024/02/28 17:31:15.590 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626535] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:15.591 [2024/02/28 17:31:15.590 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626526] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:15.591 [2024/02/28 17:31:15.590 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626523] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:15.591 [2024/02/28 17:31:15.590 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626535] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:15.591 [2024/02/28 17:31:15.590 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626530] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:15.591 [2024/02/28 17:31:15.590 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626530] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:15.591 [2024/02/28 17:31:15.590 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626523] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:15.591 [2024/02/28 17:31:15.590 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626532] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:15.591 [2024/02/28 17:31:15.590 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626532] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:15.591 [2024/02/28 17:31:15.590 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626538] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:15.591 [2024/02/28 17:31:15.590 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626538] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:15.647 [2024/02/28 17:31:15.646 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626540] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:15.647 [2024/02/28 17:31:15.646 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626534] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:15.647 [2024/02/28 17:31:15.647 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626540] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:15.647 [2024/02/28 17:31:15.647 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626534] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:15.647 [2024/02/28 17:31:15.647 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626544] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:15.647 [2024/02/28 17:31:15.647 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626544] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:15.664 [2024/02/28 17:31:15.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626553] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626524214][leader=3]"] 2024-02-29 01:31:15.664 [2024/02/28 17:31:15.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626554] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:15.664 [2024/02/28 17:31:15.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626555] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:15.664 [2024/02/28 17:31:15.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626556] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:15.664 [2024/02/28 17:31:15.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626557] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:15.664 [2024/02/28 17:31:15.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626558] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626528386][leader=3]"] 2024-02-29 01:31:15.664 [2024/02/28 17:31:15.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626559] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626527785][leader=3]"] 2024-02-29 01:31:15.665 [2024/02/28 17:31:15.665 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:15.665 [2024/02/28 17:31:15.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:15.665 [2024/02/28 17:31:15.665 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:15.665 [2024/02/28 17:31:15.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626547521] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:15.666 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:15.667 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:15.667 [2024/02/28 17:31:15.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:15.673 [2024/02/28 17:31:15.673 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626554] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:15.674 [2024/02/28 17:31:15.673 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626554] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:15.674 [2024/02/28 17:31:15.674 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626554] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:15.674 [2024/02/28 17:31:15.674 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626557] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:15.674 [2024/02/28 17:31:15.674 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626557] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:15.676 [2024/02/28 17:31:15.674 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626557] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:15.676 [2024/02/28 17:31:15.674 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626556] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:15.676 [2024/02/28 17:31:15.674 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626556] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:15.676 [2024/02/28 17:31:15.674 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626556] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:15.676 [2024/02/28 17:31:15.675 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626555] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:15.676 [2024/02/28 17:31:15.675 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626555] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:15.676 [2024/02/28 17:31:15.675 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626555] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:15.676 [2024/02/28 17:31:15.675 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626554] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:15.676 [2024/02/28 17:31:15.675 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626557] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:15.676 [2024/02/28 17:31:15.675 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626556] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:15.676 [2024/02/28 17:31:15.675 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626555] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:15.676 [2024/02/28 17:31:15.675 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=3] [toRemoveNum=4] 2024-02-29 01:31:15.676 [2024/02/28 17:31:15.675 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=40] [waitingTaskNum=0] [segmentTaskNum=40] [channelTaskNum=0] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=25] ["time cost"=516.422µs] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626530] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626530] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760748] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626519] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626519] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760025] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626526] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626526] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759628] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626538] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626538] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759770] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626518] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626518] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761400] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626544] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626544] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760374] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626523] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626523] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760744] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.677 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626540] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626540] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759849] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626542] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626542] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760199] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626533] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626533] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760422] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626534] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626534] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761572] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626535] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626535] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760696] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626532] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626532] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760192] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626522] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626522] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759971] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626543] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626543] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760215] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=25] [committedNum=3] [toRemoveNum=15] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=25] [waitingTaskNum=0] [segmentTaskNum=25] [channelTaskNum=0] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626558] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626553] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:15.687 [2024/02/28 17:31:15.679 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626559] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626531] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626525] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626537] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626525] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626521] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626527] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626521] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626537] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626527] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626529] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626536] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626529] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626536] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626516] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626516] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626539] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626528] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626539] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:15.757 [2024/02/28 17:31:15.757 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626531] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:15.758 [2024/02/28 17:31:15.757 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626517] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:15.758 [2024/02/28 17:31:15.757 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626517] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:15.758 [2024/02/28 17:31:15.757 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626541] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:15.758 [2024/02/28 17:31:15.757 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626528] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:15.758 [2024/02/28 17:31:15.757 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626541] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:15.758 [2024/02/28 17:31:15.758 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626524] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:15.758 [2024/02/28 17:31:15.758 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626524] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:15.758 [2024/02/28 17:31:15.758 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626520] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:15.758 [2024/02/28 17:31:15.758 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626520] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:15.802 [2024/02/28 17:31:15.802 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626551] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:15.802 [2024/02/28 17:31:15.802 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626551] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:15.831 [2024/02/28 17:31:15.831 +00:00] [DEBUG] [querycoordv2/server.go:528] ["QueryCoord current state"] [StateCode=Healthy] 2024-02-29 01:31:15.831 [2024/02/28 17:31:15.831 +00:00] [DEBUG] [querycoordv2/server.go:528] ["QueryCoord current state"] [StateCode=Healthy] 2024-02-29 01:31:15.859 [2024/02/28 17:31:15.858 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626550] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:15.859 [2024/02/28 17:31:15.858 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626550] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:15.868 [2024/02/28 17:31:15.868 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626552] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:15.868 [2024/02/28 17:31:15.868 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626552] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:15.898 [2024/02/28 17:31:15.898 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626553] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:15.906 [2024/02/28 17:31:15.906 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626558] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:15.911 [2024/02/28 17:31:15.911 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626558] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] [elapsed=5.274046ms] 2024-02-29 01:31:15.936 [2024/02/28 17:31:15.936 +00:00] [WARN] [task/executor.go:468] ["failed to sync distribution"] [taskID=1709136626553] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:15.936 [2024/02/28 17:31:15.936 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626553] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:15.950 [2024/02/28 17:31:15.950 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626559] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:15.955 [2024/02/28 17:31:15.955 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626559] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] [elapsed=5.511417ms] 2024-02-29 01:31:16.200 [2024/02/28 17:31:16.200 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=25] [waitingTaskNum=0] [segmentTaskNum=25] [channelTaskNum=0] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.200 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=5] ["time cost"=128.422µs] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626553] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626524214] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626559] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626527785] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626558] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626528386] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626537] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626537] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761556] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626529] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626529] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760366] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626551] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626551] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761308] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626527] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626527] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761031] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626536] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626536] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759826] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626541] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626541] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759777] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626528] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626528] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760970] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626516] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626516] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742556] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626524] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626524] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760570] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626521] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626521] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759725] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626517] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626517] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742592] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626539] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626539] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759612] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626520] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626520] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760757] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626531] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626531] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761143] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626550] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626550] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760430] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626525] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:16.201 [2024/02/28 17:31:16.201 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:16.202 [2024/02/28 17:31:16.201 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626525] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761482] 2024-02-29 01:31:16.202 [2024/02/28 17:31:16.202 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626552] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:16.202 [2024/02/28 17:31:16.202 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:16.202 [2024/02/28 17:31:16.202 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626552] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759968] 2024-02-29 01:31:16.202 [2024/02/28 17:31:16.202 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=5] [committedNum=0] [toRemoveNum=20] 2024-02-29 01:31:16.202 [2024/02/28 17:31:16.202 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=5] [waitingTaskNum=0] [segmentTaskNum=5] [channelTaskNum=0] 2024-02-29 01:31:16.585 [2024/02/28 17:31:16.584 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:16.585 [2024/02/28 17:31:16.584 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=d8fea647d291137c3925a8c83f8bc813] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:16.585 [2024/02/28 17:31:16.585 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:16.585 [2024/02/28 17:31:16.585 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=5fab23bab0a9406e4fdf1dee88ec8278] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:16.652 [2024/02/28 17:31:16.652 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:16.652 [2024/02/28 17:31:16.652 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=dadd085431d84a561c3bb28ba14cab49] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:16.663 [2024/02/28 17:31:16.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626560] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626524214][leader=3]"] 2024-02-29 01:31:16.663 [2024/02/28 17:31:16.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626561] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:16.663 [2024/02/28 17:31:16.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626562] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626528386][leader=3]"] 2024-02-29 01:31:16.663 [2024/02/28 17:31:16.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626563] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:16.663 [2024/02/28 17:31:16.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626564] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:16.664 [2024/02/28 17:31:16.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626565] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626527785][leader=3]"] 2024-02-29 01:31:16.664 [2024/02/28 17:31:16.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626566] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:16.664 [2024/02/28 17:31:16.664 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:16.664 [2024/02/28 17:31:16.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:16.664 [2024/02/28 17:31:16.664 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:16.664 [2024/02/28 17:31:16.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626547521] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.665 [2024/02/28 17:31:16.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626567] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742556]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626568] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742592]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626569] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761400]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626570] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760025]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626571] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760757]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626572] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760430]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626573] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759725]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626574] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759971]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626575] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760744]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626576] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760570]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626577] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761482]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626578] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759628]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626579] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761031]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626580] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760970]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626581] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761308]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626582] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760748]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626583] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760366]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626584] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761572]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626585] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760192]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626586] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760422]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626587] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761143]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626588] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759826]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626589] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761556]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626590] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760696]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626591] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759612]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626592] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759770]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626593] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759849]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626594] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759777]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626595] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760199]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626596] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760215]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626597] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759968]"] 2024-02-29 01:31:16.666 [2024/02/28 17:31:16.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626598] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760374]"] 2024-02-29 01:31:16.670 [2024/02/28 17:31:16.670 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626561] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:16.670 [2024/02/28 17:31:16.670 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626561] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:16.670 [2024/02/28 17:31:16.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626561] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:16.670 [2024/02/28 17:31:16.670 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626564] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:16.670 [2024/02/28 17:31:16.670 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626564] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:16.671 [2024/02/28 17:31:16.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626564] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:16.671 [2024/02/28 17:31:16.670 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626563] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:16.671 [2024/02/28 17:31:16.670 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626563] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:16.671 [2024/02/28 17:31:16.671 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626563] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:16.671 [2024/02/28 17:31:16.671 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626566] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:16.671 [2024/02/28 17:31:16.671 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626566] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:16.672 [2024/02/28 17:31:16.671 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626566] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:16.673 [2024/02/28 17:31:16.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626561] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:16.674 [2024/02/28 17:31:16.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626564] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:16.674 [2024/02/28 17:31:16.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626563] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:16.674 [2024/02/28 17:31:16.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626566] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:16.674 [2024/02/28 17:31:16.673 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=35] [toRemoveNum=4] 2024-02-29 01:31:16.674 [2024/02/28 17:31:16.673 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=40] [waitingTaskNum=0] [segmentTaskNum=40] [channelTaskNum=0] 2024-02-29 01:31:16.676 [2024/02/28 17:31:16.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626570] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.677 [2024/02/28 17:31:16.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626590] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.677 [2024/02/28 17:31:16.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626562] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:16.677 [2024/02/28 17:31:16.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626583] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.677 [2024/02/28 17:31:16.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626567] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.677 [2024/02/28 17:31:16.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626585] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.677 [2024/02/28 17:31:16.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626577] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.677 [2024/02/28 17:31:16.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626593] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.677 [2024/02/28 17:31:16.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626592] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.677 [2024/02/28 17:31:16.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626576] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.677 [2024/02/28 17:31:16.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626596] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.677 [2024/02/28 17:31:16.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626571] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.677 [2024/02/28 17:31:16.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626584] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.678 [2024/02/28 17:31:16.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626568] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.678 [2024/02/28 17:31:16.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626565] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:16.678 [2024/02/28 17:31:16.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626579] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.678 [2024/02/28 17:31:16.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626574] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.678 [2024/02/28 17:31:16.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626597] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.678 [2024/02/28 17:31:16.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626595] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.678 [2024/02/28 17:31:16.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626578] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.678 [2024/02/28 17:31:16.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626582] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.678 [2024/02/28 17:31:16.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626575] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.678 [2024/02/28 17:31:16.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626580] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.678 [2024/02/28 17:31:16.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626586] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.678 [2024/02/28 17:31:16.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626598] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.678 [2024/02/28 17:31:16.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626573] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.679 [2024/02/28 17:31:16.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626588] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.679 [2024/02/28 17:31:16.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626560] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:16.679 [2024/02/28 17:31:16.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626572] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.679 [2024/02/28 17:31:16.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626581] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.679 [2024/02/28 17:31:16.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626589] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.679 [2024/02/28 17:31:16.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626569] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.680 [2024/02/28 17:31:16.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626587] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.680 [2024/02/28 17:31:16.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626594] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.680 [2024/02/28 17:31:16.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626591] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:16.680 [2024/02/28 17:31:16.678 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=40] ["time cost"=2.950278ms] 2024-02-29 01:31:16.680 [2024/02/28 17:31:16.680 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=40] [committedNum=35] [toRemoveNum=0] 2024-02-29 01:31:16.680 [2024/02/28 17:31:16.680 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=40] [waitingTaskNum=0] [segmentTaskNum=40] [channelTaskNum=0] 2024-02-29 01:31:16.868 [2024/02/28 17:31:16.867 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:16.868 [2024/02/28 17:31:16.868 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=b511f958b82b212ba6e811b26a8226e7] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:16.888 [2024/02/28 17:31:16.887 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:16.888 [2024/02/28 17:31:16.887 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=01c87628767a6c53cfd473f6b0a0a557] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:16.896 [2024/02/28 17:31:16.896 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:16.896 [2024/02/28 17:31:16.896 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=ff8c1d79e2ea16a91f353bc2c094aa60] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:16.907 [2024/02/28 17:31:16.907 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626584] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:16.983 [2024/02/28 17:31:16.982 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:16.983 [2024/02/28 17:31:16.983 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=e34698965a03f8a73c73b0aeed67f709] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:16.987 [2024/02/28 17:31:16.986 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:16.987 [2024/02/28 17:31:16.987 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=7dc64ea27b9eb8aef99c5bdb37d9e221] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:16.990 [2024/02/28 17:31:16.990 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626569] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.014 [2024/02/28 17:31:17.014 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626593] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.052 [2024/02/28 17:31:17.052 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626580] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.052 [2024/02/28 17:31:17.052 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626568] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.080 [2024/02/28 17:31:17.080 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:17.080 [2024/02/28 17:31:17.080 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=22fc13ba5307677ce9fcfec5f5486f42] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:17.097 [2024/02/28 17:31:17.097 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626576] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.113 [2024/02/28 17:31:17.113 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626584] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:17.113 [2024/02/28 17:31:17.113 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626584] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:17.149 [2024/02/28 17:31:17.149 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626527182] 2024-02-29 01:31:17.149 [2024/02/28 17:31:17.149 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=e423fc677de09d7091d834798f5070d7] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626527182]"] 2024-02-29 01:31:17.159 [2024/02/28 17:31:17.159 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626574] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.161 [2024/02/28 17:31:17.161 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626572] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.171 [2024/02/28 17:31:17.171 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=40] [waitingTaskNum=0] [segmentTaskNum=40] [channelTaskNum=0] 2024-02-29 01:31:17.173 [2024/02/28 17:31:17.173 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=39] ["time cost"=409.881µs] 2024-02-29 01:31:17.173 [2024/02/28 17:31:17.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626584] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:17.173 [2024/02/28 17:31:17.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:17.173 [2024/02/28 17:31:17.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626584] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761572] 2024-02-29 01:31:17.173 [2024/02/28 17:31:17.173 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=39] [committedNum=0] [toRemoveNum=1] 2024-02-29 01:31:17.173 [2024/02/28 17:31:17.173 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=39] [waitingTaskNum=0] [segmentTaskNum=39] [channelTaskNum=0] 2024-02-29 01:31:17.186 [2024/02/28 17:31:17.186 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626578] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.195 [2024/02/28 17:31:17.195 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626592] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.196 [2024/02/28 17:31:17.195 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626588] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.243 [2024/02/28 17:31:17.242 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626570] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.266 [2024/02/28 17:31:17.266 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626569] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:17.266 [2024/02/28 17:31:17.266 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626569] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:17.267 [2024/02/28 17:31:17.267 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626597] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.293 [2024/02/28 17:31:17.293 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:17.294 [2024/02/28 17:31:17.293 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=06721f92fd9dc06007974831ed8437ff] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:17.295 [2024/02/28 17:31:17.295 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:17.295 [2024/02/28 17:31:17.295 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=7f05ef85d858f7a22f862c8ff8a79b44] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:17.336 [2024/02/28 17:31:17.336 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626582] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.336 [2024/02/28 17:31:17.336 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626589] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.336 [2024/02/28 17:31:17.336 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626595] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.336 [2024/02/28 17:31:17.336 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626590] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.336 [2024/02/28 17:31:17.336 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626565] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:17.336 [2024/02/28 17:31:17.336 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626583] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.340 [2024/02/28 17:31:17.339 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626565] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] [elapsed=3.294758ms] 2024-02-29 01:31:17.359 [2024/02/28 17:31:17.359 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626571] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.360 [2024/02/28 17:31:17.359 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626579] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.365 [2024/02/28 17:31:17.365 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626576] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:17.365 [2024/02/28 17:31:17.365 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626576] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:17.380 [2024/02/28 17:31:17.377 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626585] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.391 [2024/02/28 17:31:17.391 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:17.391 [2024/02/28 17:31:17.391 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=82d61ea397179decf735a21a98398a03] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:17.433 [2024/02/28 17:31:17.433 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626581] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.434 [2024/02/28 17:31:17.434 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=c1e83bb4d48eec339b2887d322d8100f] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298942,448039877626299027,448039877626298959,448039877626298936,448039877626298976,448039877626298993,448039877626299016,448039877626298999,448039877626299022,448039877626298970,448039877626299011,448039877626298931,448039877626299005,448039877626298948,448039877626298953,448039877626299033,448039877626298965,448039877626298981,448039877626298987]"] 2024-02-29 01:31:17.434 [2024/02/28 17:31:17.434 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=c1e83bb4d48eec339b2887d322d8100f] [collectionID=448039877626298924] 2024-02-29 01:31:17.434 [2024/02/28 17:31:17.434 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=c1e83bb4d48eec339b2887d322d8100f] [collectionID=448039877626298924] 2024-02-29 01:31:17.434 [2024/02/28 17:31:17.434 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:17.435 [2024/02/28 17:31:17.435 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=c1e83bb4d48eec339b2887d322d8100f] [collectionID=448039877626298924] 2024-02-29 01:31:17.435 [2024/02/28 17:31:17.435 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=c1e83bb4d48eec339b2887d322d8100f] [collectionID=448039877626298924] 2024-02-29 01:31:17.456 [2024/02/28 17:31:17.456 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626598] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.456 [2024/02/28 17:31:17.456 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626587] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.456 [2024/02/28 17:31:17.456 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626577] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.494 [2024/02/28 17:31:17.494 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626596] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.495 [2024/02/28 17:31:17.494 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626575] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.495 [2024/02/28 17:31:17.494 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626562] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:17.495 [2024/02/28 17:31:17.494 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626591] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.496 [2024/02/28 17:31:17.496 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626586] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.499 [2024/02/28 17:31:17.498 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626562] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] [elapsed=4.135219ms] 2024-02-29 01:31:17.529 [2024/02/28 17:31:17.528 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626594] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.538 [2024/02/28 17:31:17.538 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:17.538 [2024/02/28 17:31:17.538 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=ecfbb086b085a949f195cc8a828b2f58] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:17.557 [2024/02/28 17:31:17.555 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626567] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.592 [2024/02/28 17:31:17.592 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=723506a1a883546f9810db4da7dae61b] [collections="[448039877626298924]"] 2024-02-29 01:31:17.597 [2024/02/28 17:31:17.595 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:17.597 [2024/02/28 17:31:17.596 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=7ee555e1cf1389f771d9d33a6327a0ab] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:17.604 [2024/02/28 17:31:17.604 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626573] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.604 [2024/02/28 17:31:17.604 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=3c89524d55c163814f8f9fe993d4d4d6] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298987,448039877626299011,448039877626298976,448039877626298953,448039877626298970,448039877626298948,448039877626298965,448039877626298942,448039877626298993,448039877626298999,448039877626298959,448039877626299022,448039877626298931,448039877626299005,448039877626299027,448039877626298981,448039877626298936,448039877626299033,448039877626299016]"] 2024-02-29 01:31:17.604 [2024/02/28 17:31:17.604 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=3c89524d55c163814f8f9fe993d4d4d6] [collectionID=448039877626298924] 2024-02-29 01:31:17.605 [2024/02/28 17:31:17.604 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=3c89524d55c163814f8f9fe993d4d4d6] [collectionID=448039877626298924] 2024-02-29 01:31:17.605 [2024/02/28 17:31:17.604 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:17.605 [2024/02/28 17:31:17.605 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=3c89524d55c163814f8f9fe993d4d4d6] [collectionID=448039877626298924] 2024-02-29 01:31:17.605 [2024/02/28 17:31:17.605 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=3c89524d55c163814f8f9fe993d4d4d6] [collectionID=448039877626298924] 2024-02-29 01:31:17.653 [2024/02/28 17:31:17.653 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626593] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:17.653 [2024/02/28 17:31:17.653 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626568] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:17.653 [2024/02/28 17:31:17.653 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626568] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:17.653 [2024/02/28 17:31:17.653 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626593] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:17.656 [2024/02/28 17:31:17.656 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626560] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:17.663 [2024/02/28 17:31:17.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626599] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:17.663 [2024/02/28 17:31:17.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626600] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:17.663 [2024/02/28 17:31:17.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626601] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:17.663 [2024/02/28 17:31:17.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626602] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:17.664 [2024/02/28 17:31:17.664 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626523124] [error="node offline[node=8]"] 2024-02-29 01:31:17.664 [2024/02/28 17:31:17.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:17.664 [2024/02/28 17:31:17.664 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:17.664 [2024/02/28 17:31:17.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626547521] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:17.665 [2024/02/28 17:31:17.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:17.666 [2024/02/28 17:31:17.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626603] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761572]"] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626601] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626601] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626601] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626599] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626599] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626599] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626602] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626602] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626602] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626600] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626600] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626600] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626601] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626599] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626602] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626600] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:17.669 [2024/02/28 17:31:17.669 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=1] [toRemoveNum=4] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.669 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=40] [waitingTaskNum=0] [segmentTaskNum=40] [channelTaskNum=0] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626603] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=34] ["time cost"=143.341µs] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626562] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626528386] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626565] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626527785] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626568] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626568] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742592] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626569] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626569] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761400] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626593] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626593] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759849] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626576] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626576] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760570] 2024-02-29 01:31:17.670 [2024/02/28 17:31:17.670 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=34] [committedNum=1] [toRemoveNum=6] 2024-02-29 01:31:17.671 [2024/02/28 17:31:17.670 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=34] [waitingTaskNum=0] [segmentTaskNum=34] [channelTaskNum=0] 2024-02-29 01:31:17.689 [2024/02/28 17:31:17.688 +00:00] [WARN] [task/executor.go:468] ["failed to sync distribution"] [taskID=1709136626560] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:17.689 [2024/02/28 17:31:17.688 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626560] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:17.692 [2024/02/28 17:31:17.692 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:17.692 [2024/02/28 17:31:17.692 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=16ab0b1fc33538b4bda26fba82e136af] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:17.693 [2024/02/28 17:31:17.693 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=fe3d44ba08decd4bd658485bc38d7b66] [collections="[448039877626298924]"] 2024-02-29 01:31:17.693 [2024/02/28 17:31:17.693 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:17.693 [2024/02/28 17:31:17.693 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=327605b2ef7dec66e662e874ac76ab7c] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:17.695 [2024/02/28 17:31:17.695 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:17.696 [2024/02/28 17:31:17.695 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=e962568bb2aa650e6ef658ce0087a051] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:17.799 [2024/02/28 17:31:17.798 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:17.799 [2024/02/28 17:31:17.798 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=1439d0b3cc5d0fbf3400ff2bcc92b802] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:17.799 [2024/02/28 17:31:17.799 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626527182] 2024-02-29 01:31:17.799 [2024/02/28 17:31:17.799 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=c93b95addaa85a34445a082c7f649e15] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626527182]"] 2024-02-29 01:31:17.849 [2024/02/28 17:31:17.849 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:17.849 [2024/02/28 17:31:17.849 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=5c2d2789d5a9a19a91e1339f94f49f8d] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:17.947 [2024/02/28 17:31:17.947 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:17.947 [2024/02/28 17:31:17.947 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=8868221b937c20c11b65e5daa181147d] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:17.977 [2024/02/28 17:31:17.976 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626603] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:17.996 [2024/02/28 17:31:17.996 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=c3111af87f2e91c0045dff72c52283f2] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298981,448039877626298987,448039877626299022,448039877626298936,448039877626298931,448039877626298970,448039877626298953,448039877626299027,448039877626298993,448039877626298948,448039877626298976,448039877626299011,448039877626298942,448039877626298999,448039877626299005,448039877626298959,448039877626299016,448039877626299033,448039877626298965]"] 2024-02-29 01:31:17.996 [2024/02/28 17:31:17.996 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=c3111af87f2e91c0045dff72c52283f2] [collectionID=448039877626298924] 2024-02-29 01:31:17.996 [2024/02/28 17:31:17.996 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=c3111af87f2e91c0045dff72c52283f2] [collectionID=448039877626298924] 2024-02-29 01:31:17.996 [2024/02/28 17:31:17.996 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:17.997 [2024/02/28 17:31:17.997 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=c3111af87f2e91c0045dff72c52283f2] [collectionID=448039877626298924] 2024-02-29 01:31:17.997 [2024/02/28 17:31:17.997 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=c3111af87f2e91c0045dff72c52283f2] [collectionID=448039877626298924] 2024-02-29 01:31:18.005 [2024/02/28 17:31:18.005 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=4b2b06d23286dd3031c91697baa74206] [collections="[448039877626298924]"] 2024-02-29 01:31:18.099 [2024/02/28 17:31:18.099 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626574] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:18.099 [2024/02/28 17:31:18.099 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626574] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:18.112 [2024/02/28 17:31:18.112 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:18.112 [2024/02/28 17:31:18.112 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_1_448039877626298924v1 is not available in any replica"] [traceID=afc5e7971fce20e0c8f6034512786667] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:18.121 [2024/02/28 17:31:18.120 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:18.121 [2024/02/28 17:31:18.120 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=a7ec456b5796fd66a4e8aade4615fe68] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:18.125 [2024/02/28 17:31:18.125 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:18.125 [2024/02/28 17:31:18.125 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=e0533c67d46bba6a08361ac5e2254ace] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:18.169 [2024/02/28 17:31:18.169 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=34] [waitingTaskNum=0] [segmentTaskNum=34] [channelTaskNum=0] 2024-02-29 01:31:18.170 [2024/02/28 17:31:18.170 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=32] ["time cost"=143.313µs] 2024-02-29 01:31:18.170 [2024/02/28 17:31:18.170 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626560] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626524214] 2024-02-29 01:31:18.170 [2024/02/28 17:31:18.170 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626574] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:18.170 [2024/02/28 17:31:18.170 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:18.170 [2024/02/28 17:31:18.170 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626574] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759971] 2024-02-29 01:31:18.170 [2024/02/28 17:31:18.170 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=32] [committedNum=0] [toRemoveNum=2] 2024-02-29 01:31:18.170 [2024/02/28 17:31:18.170 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=32] [waitingTaskNum=0] [segmentTaskNum=32] [channelTaskNum=0] 2024-02-29 01:31:18.194 [2024/02/28 17:31:18.193 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:18.194 [2024/02/28 17:31:18.193 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=9b205e05744aa7b8c1607145859e1c63] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:18.314 [2024/02/28 17:31:18.314 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:18.314 [2024/02/28 17:31:18.314 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=5e6f0d71356a63357b87b207a91d6fb6] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:18.331 [2024/02/28 17:31:18.331 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626588] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:18.331 [2024/02/28 17:31:18.331 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626588] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:18.394 [2024/02/28 17:31:18.391 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=1999180910a8efce4935d2f2804c6c55] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298948,448039877626298942,448039877626298936,448039877626298970,448039877626298999,448039877626299011,448039877626298993,448039877626298931,448039877626299016,448039877626299027,448039877626299005,448039877626298959,448039877626299022,448039877626298976,448039877626298965,448039877626298981,448039877626299033,448039877626298953,448039877626298987]"] 2024-02-29 01:31:18.395 [2024/02/28 17:31:18.392 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=1999180910a8efce4935d2f2804c6c55] [collectionID=448039877626298924] 2024-02-29 01:31:18.395 [2024/02/28 17:31:18.392 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=1999180910a8efce4935d2f2804c6c55] [collectionID=448039877626298924] 2024-02-29 01:31:18.395 [2024/02/28 17:31:18.392 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:18.395 [2024/02/28 17:31:18.394 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=1999180910a8efce4935d2f2804c6c55] [collectionID=448039877626298924] 2024-02-29 01:31:18.395 [2024/02/28 17:31:18.394 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=1999180910a8efce4935d2f2804c6c55] [collectionID=448039877626298924] 2024-02-29 01:31:18.489 [2024/02/28 17:31:18.488 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626527182] 2024-02-29 01:31:18.489 [2024/02/28 17:31:18.488 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=c3ebf3c6c0be4c5ae0e6fd73c99e2b85] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626527182]"] 2024-02-29 01:31:18.590 [2024/02/28 17:31:18.588 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=6fd9b4b44b3d6a6b944c1d6f8b818bea] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298942,448039877626298953,448039877626298993,448039877626299016,448039877626298959,448039877626298976,448039877626298987,448039877626298999,448039877626298936,448039877626298965,448039877626298970,448039877626298931,448039877626298981,448039877626299033,448039877626299011,448039877626299022,448039877626299027,448039877626299005,448039877626298948]"] 2024-02-29 01:31:18.590 [2024/02/28 17:31:18.588 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=6fd9b4b44b3d6a6b944c1d6f8b818bea] [collectionID=448039877626298924] 2024-02-29 01:31:18.590 [2024/02/28 17:31:18.588 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=6fd9b4b44b3d6a6b944c1d6f8b818bea] [collectionID=448039877626298924] 2024-02-29 01:31:18.590 [2024/02/28 17:31:18.588 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:18.593 [2024/02/28 17:31:18.592 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=6fd9b4b44b3d6a6b944c1d6f8b818bea] [collectionID=448039877626298924] 2024-02-29 01:31:18.594 [2024/02/28 17:31:18.592 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=6fd9b4b44b3d6a6b944c1d6f8b818bea] [collectionID=448039877626298924] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626604] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626527785][leader=3]"] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626605] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.664 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626523124] [error="node offline[node=8]"] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.664 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626606] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.664 +00:00] [INFO] [task/scheduler.go:388] ["replace old task, the new one with higher priority"] [oldID=1709136625288] [oldPriority=Normal] [newID=0] [newPriority=High] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.664 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136625288] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626547521] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626607] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626547521][leader=3]"] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626608] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626528386][leader=3]"] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.665 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626609] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.665 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626610] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.665 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626611] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626524214][leader=3]"] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.665 +00:00] [WARN] [grpcclient/client.go:408] ["call received grpc error"] [clientRole=querynode-3] [error="rpc error: code = Canceled desc = context canceled"] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.665 +00:00] [WARN] [retry/retry.go:46] ["retry func failed"] [retried=0] [error="rpc error: code = Canceled desc = context canceled"] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.665 +00:00] [WARN] [grpcclient/client.go:553] ["ClientBase Call grpc call get error"] [role=querynode-3] [address=10.104.5.220:21123] [error="stack trace: /go/src/github.com/milvus-io/milvus/pkg/tracer/stack_trace.go:51 github.com/milvus-io/milvus/pkg/tracer.StackTrace\n/go/src/github.com/milvus-io/milvus/internal/util/grpcclient/client.go:552 github.com/milvus-io/milvus/internal/util/grpcclient.(*ClientBase[...]).Call\n/go/src/github.com/milvus-io/milvus/internal/util/grpcclient/client.go:566 github.com/milvus-io/milvus/internal/util/grpcclient.(*ClientBase[...]).ReCall\n/go/src/github.com/milvus-io/milvus/internal/distributed/querynode/client/client.go:90 github.com/milvus-io/milvus/internal/distributed/querynode/client.wrapGrpcCall[...]\n/go/src/github.com/milvus-io/milvus/internal/distributed/querynode/client/client.go:151 github.com/milvus-io/milvus/internal/distributed/querynode/client.(*Client).LoadSegments\n/go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:122 github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).LoadSegments.func1\n/go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:271 github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).send\n/go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:119 github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).LoadSegments\n/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/executor.go:203 github.com/milvus-io/milvus/internal/querycoordv2/task.(*Executor).loadSegment\n/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/executor.go:141 github.com/milvus-io/milvus/internal/querycoordv2/task.(*Executor).executeSegmentAction: rpc error: code = Canceled desc = context canceled"] [errorVerbose="stack trace: /go/src/github.com/milvus-io/milvus/pkg/tracer/stack_trace.go:51 github.com/milvus-io/milvus/pkg/tracer.StackTrace: rpc error: code = Canceled desc = context canceled\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/internal/util/grpcclient.(*ClientBase[...]).Call\n | \t/go/src/github.com/milvus-io/milvus/internal/util/grpcclient/client.go:552\n | github.com/milvus-io/milvus/internal/util/grpcclient.(*ClientBase[...]).ReCall\n | \t/go/src/github.com/milvus-io/milvus/internal/util/grpcclient/client.go:566\n | github.com/milvus-io/milvus/internal/distributed/querynode/client.wrapGrpcCall[...]\n | \t/go/src/github.com/milvus-io/milvus/internal/distributed/querynode/client/client.go:90\n | github.com/milvus-io/milvus/internal/distributed/querynode/client.(*Client).LoadSegments\n | \t/go/src/github.com/milvus-io/milvus/internal/distributed/querynode/client/client.go:151\n | github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).LoadSegments.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:122\n | github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).send\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:271\n | github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).LoadSegments\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:119\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*Executor).loadSegment\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/executor.go:203\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*Executor).executeSegmentAction\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/executor.go:141\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*Executor).Execute.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/executor.go:113\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) stack trace: /go/src/github.com/milvus-io/milvus/pkg/tracer/stack_trace.go:51 github.com/milvus-io/milvus/pkg/tracer.StackTrace\n | /go/src/github.com/milvus-io/milvus/internal/util/grpcclient/client.go:552 github.com/milvus-io/milvus/internal/util/grpcclient.(*ClientBase[...]).Call\n | /go/src/github.com/milvus-io/milvus/internal/util/grpcclient/client.go:566 github.com/milvus-io/milvus/internal/util/grpcclient.(*ClientBase[...]).ReCall\n | /go/src/github.com/milvus-io/milvus/internal/distributed/querynode/client/client.go:90 github.com/milvus-io/milvus/internal/distributed/querynode/client.wrapGrpcCall[...]\n | /go/src/github.com/milvus-io/milvus/internal/distributed/querynode/client/client.go:151 github.com/milvus-io/milvus/internal/distributed/querynode/client.(*Client).LoadSegments\n | /go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:122 github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).LoadSegments.func1\n | /go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:271 github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).send\n | /go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:119 github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).LoadSegments\n | /go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/executor.go:203 github.com/milvus-io/milvus/internal/querycoordv2/task.(*Executor).loadSegment\n | /go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/executor.go:141 github.com/milvus-io/milvus/internal/querycoordv2/task.(*Executor).executeSegmentAction\nWraps: (3) rpc error: code = Canceled desc = context canceled\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) *status.Error"] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.665 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136625288] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626547521] [node=3] [source=segment_checker] [shardLeader=3] [error="stack trace: /go/src/github.com/milvus-io/milvus/pkg/tracer/stack_trace.go:51 github.com/milvus-io/milvus/pkg/tracer.StackTrace\n/go/src/github.com/milvus-io/milvus/internal/util/grpcclient/client.go:552 github.com/milvus-io/milvus/internal/util/grpcclient.(*ClientBase[...]).Call\n/go/src/github.com/milvus-io/milvus/internal/util/grpcclient/client.go:566 github.com/milvus-io/milvus/internal/util/grpcclient.(*ClientBase[...]).ReCall\n/go/src/github.com/milvus-io/milvus/internal/distributed/querynode/client/client.go:90 github.com/milvus-io/milvus/internal/distributed/querynode/client.wrapGrpcCall[...]\n/go/src/github.com/milvus-io/milvus/internal/distributed/querynode/client/client.go:151 github.com/milvus-io/milvus/internal/distributed/querynode/client.(*Client).LoadSegments\n/go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:122 github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).LoadSegments.func1\n/go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:271 github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).send\n/go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:119 github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).LoadSegments\n/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/executor.go:203 github.com/milvus-io/milvus/internal/querycoordv2/task.(*Executor).loadSegment\n/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/executor.go:141 github.com/milvus-io/milvus/internal/querycoordv2/task.(*Executor).executeSegmentAction: rpc error: code = Canceled desc = context canceled"] [errorVerbose="stack trace: /go/src/github.com/milvus-io/milvus/pkg/tracer/stack_trace.go:51 github.com/milvus-io/milvus/pkg/tracer.StackTrace: rpc error: code = Canceled desc = context canceled\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/internal/util/grpcclient.(*ClientBase[...]).Call\n | \t/go/src/github.com/milvus-io/milvus/internal/util/grpcclient/client.go:552\n | github.com/milvus-io/milvus/internal/util/grpcclient.(*ClientBase[...]).ReCall\n | \t/go/src/github.com/milvus-io/milvus/internal/util/grpcclient/client.go:566\n | github.com/milvus-io/milvus/internal/distributed/querynode/client.wrapGrpcCall[...]\n | \t/go/src/github.com/milvus-io/milvus/internal/distributed/querynode/client/client.go:90\n | github.com/milvus-io/milvus/internal/distributed/querynode/client.(*Client).LoadSegments\n | \t/go/src/github.com/milvus-io/milvus/internal/distributed/querynode/client/client.go:151\n | github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).LoadSegments.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:122\n | github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).send\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:271\n | github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).LoadSegments\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:119\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*Executor).loadSegment\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/executor.go:203\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*Executor).executeSegmentAction\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/executor.go:141\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*Executor).Execute.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/executor.go:113\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) stack trace: /go/src/github.com/milvus-io/milvus/pkg/tracer/stack_trace.go:51 github.com/milvus-io/milvus/pkg/tracer.StackTrace\n | /go/src/github.com/milvus-io/milvus/internal/util/grpcclient/client.go:552 github.com/milvus-io/milvus/internal/util/grpcclient.(*ClientBase[...]).Call\n | /go/src/github.com/milvus-io/milvus/internal/util/grpcclient/client.go:566 github.com/milvus-io/milvus/internal/util/grpcclient.(*ClientBase[...]).ReCall\n | /go/src/github.com/milvus-io/milvus/internal/distributed/querynode/client/client.go:90 github.com/milvus-io/milvus/internal/distributed/querynode/client.wrapGrpcCall[...]\n | /go/src/github.com/milvus-io/milvus/internal/distributed/querynode/client/client.go:151 github.com/milvus-io/milvus/internal/distributed/querynode/client.(*Client).LoadSegments\n | /go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:122 github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).LoadSegments.func1\n | /go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:271 github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).send\n | /go/src/github.com/milvus-io/milvus/internal/querycoordv2/session/cluster.go:119 github.com/milvus-io/milvus/internal/querycoordv2/session.(*QueryCluster).LoadSegments\n | /go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/executor.go:203 github.com/milvus-io/milvus/internal/querycoordv2/task.(*Executor).loadSegment\n | /go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/executor.go:141 github.com/milvus-io/milvus/internal/querycoordv2/task.(*Executor).executeSegmentAction\nWraps: (3) rpc error: code = Canceled desc = context canceled\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) *status.Error"] 2024-02-29 01:31:18.665 [2024/02/28 17:31:18.665 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136625288] [step=0] [error="service internal error: replaced with the other one with higher priority"] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.666 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.667 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.667 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.667 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.667 [2024/02/28 17:31:18.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.667 [2024/02/28 17:31:18.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.667 [2024/02/28 17:31:18.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:18.667 [2024/02/28 17:31:18.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:18.667 [2024/02/28 17:31:18.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:18.667 [2024/02/28 17:31:18.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:18.667 [2024/02/28 17:31:18.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:18.668 [2024/02/28 17:31:18.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626612] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761400]"] 2024-02-29 01:31:18.668 [2024/02/28 17:31:18.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626613] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759971]"] 2024-02-29 01:31:18.668 [2024/02/28 17:31:18.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626614] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760570]"] 2024-02-29 01:31:18.668 [2024/02/28 17:31:18.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626615] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759849]"] 2024-02-29 01:31:18.668 [2024/02/28 17:31:18.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626616] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742592]"] 2024-02-29 01:31:18.680 [2024/02/28 17:31:18.678 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626605] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:18.680 [2024/02/28 17:31:18.678 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626605] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:18.680 [2024/02/28 17:31:18.679 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626605] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:18.680 [2024/02/28 17:31:18.679 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626609] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:18.680 [2024/02/28 17:31:18.679 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626609] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:18.680 [2024/02/28 17:31:18.679 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626609] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:18.680 [2024/02/28 17:31:18.679 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626606] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:18.683 [2024/02/28 17:31:18.680 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626606] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:18.683 [2024/02/28 17:31:18.680 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626606] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:18.683 [2024/02/28 17:31:18.680 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626607] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626547521] [taskType=Grow] 2024-02-29 01:31:18.683 [2024/02/28 17:31:18.680 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626607] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626547521]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626547521]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626547521]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:18.683 [2024/02/28 17:31:18.680 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626607] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626547521]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626547521]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626547521]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:18.683 [2024/02/28 17:31:18.681 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626610] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:18.683 [2024/02/28 17:31:18.681 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626610] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:18.683 [2024/02/28 17:31:18.681 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626610] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:18.683 [2024/02/28 17:31:18.681 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626605] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:18.683 [2024/02/28 17:31:18.681 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626609] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:18.683 [2024/02/28 17:31:18.681 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626606] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:18.683 [2024/02/28 17:31:18.682 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626607] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626547521] 2024-02-29 01:31:18.683 [2024/02/28 17:31:18.682 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626610] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:18.683 [2024/02/28 17:31:18.682 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=8] [toRemoveNum=5] 2024-02-29 01:31:18.683 [2024/02/28 17:31:18.682 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=39] [waitingTaskNum=0] [segmentTaskNum=39] [channelTaskNum=0] 2024-02-29 01:31:18.685 [2024/02/28 17:31:18.685 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626608] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:18.686 [2024/02/28 17:31:18.685 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=38] ["time cost"=1.071821ms] 2024-02-29 01:31:18.686 [2024/02/28 17:31:18.685 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626588] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:18.686 [2024/02/28 17:31:18.686 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:18.687 [2024/02/28 17:31:18.686 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626588] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759826] 2024-02-29 01:31:18.687 [2024/02/28 17:31:18.686 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=38] [committedNum=8] [toRemoveNum=1] 2024-02-29 01:31:18.687 [2024/02/28 17:31:18.686 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=38] [waitingTaskNum=0] [segmentTaskNum=38] [channelTaskNum=0] 2024-02-29 01:31:18.687 [2024/02/28 17:31:18.686 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626615] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:18.687 [2024/02/28 17:31:18.686 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626611] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:18.687 [2024/02/28 17:31:18.686 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626612] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:18.690 [2024/02/28 17:31:18.687 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626604] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:18.690 [2024/02/28 17:31:18.687 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626616] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:18.690 [2024/02/28 17:31:18.687 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626572] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:18.690 [2024/02/28 17:31:18.687 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626572] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:18.690 [2024/02/28 17:31:18.687 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626580] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:18.690 [2024/02/28 17:31:18.687 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626580] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:18.690 [2024/02/28 17:31:18.687 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626613] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:18.690 [2024/02/28 17:31:18.688 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626570] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:18.690 [2024/02/28 17:31:18.688 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626570] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:18.690 [2024/02/28 17:31:18.688 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626614] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:18.716 [2024/02/28 17:31:18.716 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=f7776b224f558dac4c428b1df6883b77] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298948,448039877626298987,448039877626298959,448039877626298999,448039877626298942,448039877626299016,448039877626299022,448039877626298981,448039877626298965,448039877626299033,448039877626298953,448039877626298931,448039877626299027,448039877626299011,448039877626299005,448039877626298976,448039877626298993,448039877626298970,448039877626298936]"] 2024-02-29 01:31:18.716 [2024/02/28 17:31:18.716 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=f7776b224f558dac4c428b1df6883b77] [collectionID=448039877626298924] 2024-02-29 01:31:18.716 [2024/02/28 17:31:18.716 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=f7776b224f558dac4c428b1df6883b77] [collectionID=448039877626298924] 2024-02-29 01:31:18.716 [2024/02/28 17:31:18.716 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:18.717 [2024/02/28 17:31:18.717 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=f7776b224f558dac4c428b1df6883b77] [collectionID=448039877626298924] 2024-02-29 01:31:18.717 [2024/02/28 17:31:18.717 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=f7776b224f558dac4c428b1df6883b77] [collectionID=448039877626298924] 2024-02-29 01:31:18.718 [2024/02/28 17:31:18.718 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=66bffead2237ec90de7451d36e2459c9] [collections="[448039877626298924]"] 2024-02-29 01:31:18.720 [2024/02/28 17:31:18.719 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=0b788c85ccdad4c87841ec17187023b4] [collections="[448039877626298924]"] 2024-02-29 01:31:18.720 [2024/02/28 17:31:18.720 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:18.720 [2024/02/28 17:31:18.720 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=9576beece457b0448cf1fa1058a0a89a] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:18.796 [2024/02/28 17:31:18.795 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626578] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:18.796 [2024/02/28 17:31:18.795 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626578] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:18.796 [2024/02/28 17:31:18.795 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626592] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:18.796 [2024/02/28 17:31:18.795 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626592] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:18.817 [2024/02/28 17:31:18.817 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626523124] [error="node offline[node=8]"] 2024-02-29 01:31:18.817 [2024/02/28 17:31:18.817 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_1_448039877626298924v1 is not available in any replica"] [traceID=e814dd80026dc7f6304e3bcd4f870512] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:18.818 [2024/02/28 17:31:18.818 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:18.818 [2024/02/28 17:31:18.818 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=f256377b299648761ef9affd2fa12635] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:18.838 [2024/02/28 17:31:18.837 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626589] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:18.838 [2024/02/28 17:31:18.837 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626582] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:18.838 [2024/02/28 17:31:18.837 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626589] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:18.838 [2024/02/28 17:31:18.838 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626582] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:18.838 [2024/02/28 17:31:18.837 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626590] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:18.838 [2024/02/28 17:31:18.837 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626597] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:18.838 [2024/02/28 17:31:18.838 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626597] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:18.838 [2024/02/28 17:31:18.838 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626590] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:18.838 [2024/02/28 17:31:18.838 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626571] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:18.838 [2024/02/28 17:31:18.838 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626571] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:18.838 [2024/02/28 17:31:18.838 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626579] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:18.838 [2024/02/28 17:31:18.838 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626579] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:18.838 [2024/02/28 17:31:18.838 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626583] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:18.838 [2024/02/28 17:31:18.838 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626595] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:18.838 [2024/02/28 17:31:18.838 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626583] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:18.838 [2024/02/28 17:31:18.838 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626595] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.903 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626567] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.903 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626585] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.903 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626567] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.903 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626585] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.903 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626577] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.903 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626591] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.903 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626581] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.903 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626577] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.903 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626591] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.904 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626581] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.904 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626587] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.904 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626587] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.904 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626598] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.904 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626598] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.904 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626575] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.904 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626575] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.904 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626596] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:18.904 [2024/02/28 17:31:18.904 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626596] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:18.918 [2024/02/28 17:31:18.918 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:18.918 [2024/02/28 17:31:18.918 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=d5457485eefdeb828dd20f7686c6dd07] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:18.937 [2024/02/28 17:31:18.937 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626573] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:18.937 [2024/02/28 17:31:18.937 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626573] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:18.991 [2024/02/28 17:31:18.991 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626615] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:18.991 [2024/02/28 17:31:18.991 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626612] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:18.999 [2024/02/28 17:31:18.997 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=41ac1da1685e7b5e188730d2ca852e0e] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298987,448039877626298970,448039877626298981,448039877626298965,448039877626299022,448039877626299016,448039877626298931,448039877626298953,448039877626299011,448039877626298959,448039877626298976,448039877626299033,448039877626299005,448039877626298999,448039877626298993,448039877626298948,448039877626298942,448039877626298936,448039877626299027]"] 2024-02-29 01:31:18.999 [2024/02/28 17:31:18.998 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=41ac1da1685e7b5e188730d2ca852e0e] [collectionID=448039877626298924] 2024-02-29 01:31:18.999 [2024/02/28 17:31:18.998 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=41ac1da1685e7b5e188730d2ca852e0e] [collectionID=448039877626298924] 2024-02-29 01:31:18.999 [2024/02/28 17:31:18.998 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:18.999 [2024/02/28 17:31:18.999 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=41ac1da1685e7b5e188730d2ca852e0e] [collectionID=448039877626298924] 2024-02-29 01:31:18.999 [2024/02/28 17:31:18.999 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=41ac1da1685e7b5e188730d2ca852e0e] [collectionID=448039877626298924] 2024-02-29 01:31:19.002 [2024/02/28 17:31:19.002 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626603] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:19.002 [2024/02/28 17:31:19.002 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626603] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:19.008 [2024/02/28 17:31:19.008 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626613] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:19.008 [2024/02/28 17:31:19.008 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626614] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:19.022 [2024/02/28 17:31:19.022 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:19.022 [2024/02/28 17:31:19.022 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=f38403135410c79c7a28714aef51da47] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:19.025 [2024/02/28 17:31:19.025 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626604] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:19.025 [2024/02/28 17:31:19.025 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626616] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:19.028 [2024/02/28 17:31:19.028 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626604] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] [elapsed=2.898638ms] 2024-02-29 01:31:19.054 [2024/02/28 17:31:19.054 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626608] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:19.057 [2024/02/28 17:31:19.057 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626608] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] [elapsed=2.938803ms] 2024-02-29 01:31:19.067 [2024/02/28 17:31:19.067 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626586] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:19.067 [2024/02/28 17:31:19.067 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626586] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:19.081 [2024/02/28 17:31:19.073 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626611] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:19.129 [2024/02/28 17:31:19.129 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:19.129 [2024/02/28 17:31:19.129 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=adfd28c466a7aaf9e0e8f6e03e3a2b65] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:19.131 [2024/02/28 17:31:19.131 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=26392f28f4ea1370cb06d044fc202b73] [collections="[448039877626298924]"] 2024-02-29 01:31:19.134 [2024/02/28 17:31:19.134 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:19.134 [2024/02/28 17:31:19.134 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=6bd5e8179b5cf12bf7d32ae94b9d206d] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:19.137 [2024/02/28 17:31:19.136 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:19.137 [2024/02/28 17:31:19.136 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=82ea4046d413f64153529e39dd17bcd7] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:19.137 [2024/02/28 17:31:19.137 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=8653723e19211acba609a7ca4e47157d] [collections="[448039877626298924]"] 2024-02-29 01:31:19.138 [2024/02/28 17:31:19.138 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:19.138 [2024/02/28 17:31:19.138 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=e13f1fa546f106c641a213dab72f84c2] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:19.144 [2024/02/28 17:31:19.144 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:19.144 [2024/02/28 17:31:19.144 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=63f733e57eba7ba6da283e6c8fe020a2] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:19.145 [2024/02/28 17:31:19.145 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:19.145 [2024/02/28 17:31:19.145 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=ca695038277c7475f44020e2f0ce6cb2] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:19.175 [2024/02/28 17:31:19.174 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=38] [waitingTaskNum=0] [segmentTaskNum=38] [channelTaskNum=0] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.175 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=11] ["time cost"=281.72µs] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626608] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626528386] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626604] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626527785] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626582] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626582] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760748] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626590] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626590] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760696] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626597] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626597] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759968] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626596] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626596] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760215] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626603] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626603] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761572] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626570] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626570] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760025] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626592] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626592] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759770] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626577] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626577] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761482] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626573] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:19.176 [2024/02/28 17:31:19.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626573] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759725] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626598] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626598] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760374] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626589] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626589] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761556] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626581] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626581] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761308] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626572] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626572] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760430] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626591] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.176 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.176 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626591] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759612] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.176 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626583] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626583] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760366] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626586] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626586] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760422] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626587] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626587] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761143] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626567] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626567] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742556] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626579] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626579] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761031] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626571] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626571] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760757] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626595] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626595] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760199] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626575] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626575] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760744] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626578] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626578] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759628] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626585] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626585] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760192] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626580] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626580] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760970] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=11] [committedNum=0] [toRemoveNum=27] 2024-02-29 01:31:19.177 [2024/02/28 17:31:19.177 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=11] [waitingTaskNum=0] [segmentTaskNum=11] [channelTaskNum=0] 2024-02-29 01:31:19.205 [2024/02/28 17:31:19.204 +00:00] [WARN] [task/executor.go:468] ["failed to sync distribution"] [taskID=1709136626611] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:19.205 [2024/02/28 17:31:19.204 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626611] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:19.211 [2024/02/28 17:31:19.211 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626594] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:19.211 [2024/02/28 17:31:19.211 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626594] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:19.258 [2024/02/28 17:31:19.258 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:19.258 [2024/02/28 17:31:19.258 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_1_448039877626298924v1 is not available in any replica"] [traceID=381d01711349fe370862597af4af4803] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:19.307 [2024/02/28 17:31:19.306 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=0123d2478aae6f37aad9ded9ea596a5f] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298987,448039877626299027,448039877626298965,448039877626299022,448039877626298959,448039877626298931,448039877626298948,448039877626298993,448039877626298999,448039877626299011,448039877626298976,448039877626298970,448039877626299005,448039877626299016,448039877626298942,448039877626298953,448039877626298936,448039877626299033,448039877626298981]"] 2024-02-29 01:31:19.307 [2024/02/28 17:31:19.307 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=0123d2478aae6f37aad9ded9ea596a5f] [collectionID=448039877626298924] 2024-02-29 01:31:19.307 [2024/02/28 17:31:19.307 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=0123d2478aae6f37aad9ded9ea596a5f] [collectionID=448039877626298924] 2024-02-29 01:31:19.307 [2024/02/28 17:31:19.307 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:19.307 [2024/02/28 17:31:19.307 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=0123d2478aae6f37aad9ded9ea596a5f] [collectionID=448039877626298924] 2024-02-29 01:31:19.307 [2024/02/28 17:31:19.307 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=0123d2478aae6f37aad9ded9ea596a5f] [collectionID=448039877626298924] 2024-02-29 01:31:19.327 [2024/02/28 17:31:19.327 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:19.327 [2024/02/28 17:31:19.327 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=959d1896268194a16190d461a2a2de4e] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:19.440 [2024/02/28 17:31:19.440 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526789] 2024-02-29 01:31:19.440 [2024/02/28 17:31:19.440 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=dcbd2eac489a427e48c56efb07bc8138] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526789]"] 2024-02-29 01:31:19.440 [2024/02/28 17:31:19.440 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:19.440 [2024/02/28 17:31:19.440 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=0debe1d5844583e1ed032f2a72a35e89] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:19.459 [2024/02/28 17:31:19.459 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626615] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:19.459 [2024/02/28 17:31:19.459 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626615] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:19.469 [2024/02/28 17:31:19.469 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626616] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:19.469 [2024/02/28 17:31:19.469 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626612] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:19.469 [2024/02/28 17:31:19.469 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626616] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:19.469 [2024/02/28 17:31:19.469 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626612] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:19.469 [2024/02/28 17:31:19.469 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626613] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:19.469 [2024/02/28 17:31:19.469 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626613] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:19.482 [2024/02/28 17:31:19.481 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626614] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:19.482 [2024/02/28 17:31:19.481 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626614] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:19.490 [2024/02/28 17:31:19.489 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=bbdf52b08a5de892d4ce2f61f2f41422] [collections="[448039877626298924]"] 2024-02-29 01:31:19.490 [2024/02/28 17:31:19.490 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:19.490 [2024/02/28 17:31:19.490 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=dcaf76fecf408ffab39781c48804126d] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:19.593 [2024/02/28 17:31:19.592 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626523124] [error="node offline[node=8]"] 2024-02-29 01:31:19.593 [2024/02/28 17:31:19.592 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_1_448039877626298924v1 is not available in any replica"] [traceID=fad6b30aede56c924abf29e200743689] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:19.593 [2024/02/28 17:31:19.593 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:19.593 [2024/02/28 17:31:19.593 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=0797354b7fdfaa91324ed74f9b72e217] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:19.664 [2024/02/28 17:31:19.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626617] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626528386][leader=3]"] 2024-02-29 01:31:19.664 [2024/02/28 17:31:19.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626618] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:19.664 [2024/02/28 17:31:19.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626619] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:19.664 [2024/02/28 17:31:19.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626620] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626527785][leader=3]"] 2024-02-29 01:31:19.664 [2024/02/28 17:31:19.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626621] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:19.664 [2024/02/28 17:31:19.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626622] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:19.664 [2024/02/28 17:31:19.664 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:19.664 [2024/02/28 17:31:19.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:19.664 [2024/02/28 17:31:19.664 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:19.664 [2024/02/28 17:31:19.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526789] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626552811] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.665 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.666 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:19.666 [2024/02/28 17:31:19.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:19.666 [2024/02/28 17:31:19.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:19.666 [2024/02/28 17:31:19.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:19.666 [2024/02/28 17:31:19.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:19.666 [2024/02/28 17:31:19.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626623] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760025]"] 2024-02-29 01:31:19.666 [2024/02/28 17:31:19.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626624] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760757]"] 2024-02-29 01:31:19.666 [2024/02/28 17:31:19.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626625] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760430]"] 2024-02-29 01:31:19.666 [2024/02/28 17:31:19.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626626] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759725]"] 2024-02-29 01:31:19.666 [2024/02/28 17:31:19.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626627] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760744]"] 2024-02-29 01:31:19.666 [2024/02/28 17:31:19.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626628] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761482]"] 2024-02-29 01:31:19.666 [2024/02/28 17:31:19.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626629] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759628]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626630] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761031]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626631] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760970]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626632] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761308]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626633] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760748]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626634] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760366]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626635] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761143]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626636] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760422]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626637] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761572]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626638] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760192]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626639] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759826]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626640] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760696]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626641] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761556]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626642] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759612]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626643] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759770]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626644] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760215]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626645] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760199]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626646] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759968]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626647] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760374]"] 2024-02-29 01:31:19.667 [2024/02/28 17:31:19.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626648] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742556]"] 2024-02-29 01:31:19.673 [2024/02/28 17:31:19.673 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626619] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:19.673 [2024/02/28 17:31:19.673 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626619] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:19.673 [2024/02/28 17:31:19.673 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626619] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:19.673 [2024/02/28 17:31:19.673 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626621] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:19.673 [2024/02/28 17:31:19.673 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626621] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:19.673 [2024/02/28 17:31:19.673 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626621] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:19.673 [2024/02/28 17:31:19.673 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626618] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:19.674 [2024/02/28 17:31:19.673 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626618] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:19.674 [2024/02/28 17:31:19.674 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626618] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:19.674 [2024/02/28 17:31:19.674 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626622] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:19.674 [2024/02/28 17:31:19.674 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626622] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:19.674 [2024/02/28 17:31:19.674 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626622] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:19.675 [2024/02/28 17:31:19.675 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626619] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:19.675 [2024/02/28 17:31:19.675 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626621] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:19.675 [2024/02/28 17:31:19.675 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626618] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:19.675 [2024/02/28 17:31:19.675 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626622] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:19.675 [2024/02/28 17:31:19.675 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=28] [toRemoveNum=4] 2024-02-29 01:31:19.675 [2024/02/28 17:31:19.675 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=39] [waitingTaskNum=0] [segmentTaskNum=39] [channelTaskNum=0] 2024-02-29 01:31:19.677 [2024/02/28 17:31:19.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626627] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.677 [2024/02/28 17:31:19.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626645] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.677 [2024/02/28 17:31:19.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626629] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.677 [2024/02/28 17:31:19.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626617] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:19.677 [2024/02/28 17:31:19.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626637] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.677 [2024/02/28 17:31:19.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626641] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.677 [2024/02/28 17:31:19.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626626] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.677 [2024/02/28 17:31:19.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626624] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.678 [2024/02/28 17:31:19.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626632] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.678 [2024/02/28 17:31:19.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626648] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.678 [2024/02/28 17:31:19.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626643] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.678 [2024/02/28 17:31:19.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626635] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.678 [2024/02/28 17:31:19.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626639] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.678 [2024/02/28 17:31:19.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626646] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.678 [2024/02/28 17:31:19.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626642] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.679 [2024/02/28 17:31:19.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626633] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.679 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626631] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626647] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626634] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626620] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=32] ["time cost"=2.611144ms] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626638] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626611] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626524214] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626636] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626630] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626594] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626594] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759777] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626616] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626625] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626644] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626640] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626628] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626616] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742592] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626615] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626623] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626615] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759849] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626612] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626612] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761400] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626613] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626613] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759971] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626614] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626614] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760570] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=32] [committedNum=28] [toRemoveNum=7] 2024-02-29 01:31:19.680 [2024/02/28 17:31:19.679 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=32] [waitingTaskNum=0] [segmentTaskNum=32] [channelTaskNum=0] 2024-02-29 01:31:19.733 [2024/02/28 17:31:19.733 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:19.733 [2024/02/28 17:31:19.733 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_1_448039877626298924v1 is not available in any replica"] [traceID=45d9172553300be7b810d6bf0d538e28] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:19.753 [2024/02/28 17:31:19.753 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:19.753 [2024/02/28 17:31:19.753 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=355faa45b96acbd9c69854d965def87e] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:19.758 [2024/02/28 17:31:19.758 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:19.758 [2024/02/28 17:31:19.758 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=3fca4f6c5679b0932c1154dea85da721] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:19.950 [2024/02/28 17:31:19.950 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=4ffa7b105db514f75b7a7915a0015b42] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298981,448039877626299011,448039877626298965,448039877626299005,448039877626298987,448039877626298993,448039877626298936,448039877626299033,448039877626299016,448039877626298931,448039877626298942,448039877626298976,448039877626298970,448039877626298999,448039877626299022,448039877626298953,448039877626299027,448039877626298948,448039877626298959]"] 2024-02-29 01:31:19.950 [2024/02/28 17:31:19.950 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=4ffa7b105db514f75b7a7915a0015b42] [collectionID=448039877626298924] 2024-02-29 01:31:19.950 [2024/02/28 17:31:19.950 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=4ffa7b105db514f75b7a7915a0015b42] [collectionID=448039877626298924] 2024-02-29 01:31:19.950 [2024/02/28 17:31:19.950 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:19.951 [2024/02/28 17:31:19.951 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=4ffa7b105db514f75b7a7915a0015b42] [collectionID=448039877626298924] 2024-02-29 01:31:19.951 [2024/02/28 17:31:19.951 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=4ffa7b105db514f75b7a7915a0015b42] [collectionID=448039877626298924] 2024-02-29 01:31:19.957 [2024/02/28 17:31:19.957 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=30eeebccff9ebcb34080def5f3e19e52] [collections="[448039877626298924]"] 2024-02-29 01:31:20.039 [2024/02/28 17:31:20.039 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626637] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.084 [2024/02/28 17:31:20.083 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626623] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.089 [2024/02/28 17:31:20.089 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:20.089 [2024/02/28 17:31:20.089 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=4fd0ac485e8fbd533f000e08232ca7a5] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:20.091 [2024/02/28 17:31:20.090 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:20.091 [2024/02/28 17:31:20.090 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=675c33bc5a2dc1cf1a2d58a66938927a] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:20.129 [2024/02/28 17:31:20.129 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626641] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.129 [2024/02/28 17:31:20.129 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626634] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.130 [2024/02/28 17:31:20.130 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626640] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.130 [2024/02/28 17:31:20.130 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626629] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.130 [2024/02/28 17:31:20.130 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626647] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.131 [2024/02/28 17:31:20.131 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626628] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.131 [2024/02/28 17:31:20.131 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=d62a28f45cda359ae90602b7c05e19b3] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626299005,448039877626298976,448039877626299016,448039877626299011,448039877626298981,448039877626298970,448039877626299027,448039877626298993,448039877626298987,448039877626298999,448039877626299022,448039877626298959,448039877626298948,448039877626298965,448039877626298931,448039877626299033,448039877626298953,448039877626298936,448039877626298942]"] 2024-02-29 01:31:20.131 [2024/02/28 17:31:20.131 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=d62a28f45cda359ae90602b7c05e19b3] [collectionID=448039877626298924] 2024-02-29 01:31:20.131 [2024/02/28 17:31:20.131 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=d62a28f45cda359ae90602b7c05e19b3] [collectionID=448039877626298924] 2024-02-29 01:31:20.131 [2024/02/28 17:31:20.131 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:20.132 [2024/02/28 17:31:20.132 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=d62a28f45cda359ae90602b7c05e19b3] [collectionID=448039877626298924] 2024-02-29 01:31:20.132 [2024/02/28 17:31:20.132 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=d62a28f45cda359ae90602b7c05e19b3] [collectionID=448039877626298924] 2024-02-29 01:31:20.151 [2024/02/28 17:31:20.151 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626639] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.158 [2024/02/28 17:31:20.158 +00:00] [INFO] [task/executor.go:211] ["load segments done"] [taskID=1709136625286] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626526789] [node=3] [source=segment_checker] [shardLeader=3] [elapsed=36.546744176s] 2024-02-29 01:31:20.172 [2024/02/28 17:31:20.172 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=32] [waitingTaskNum=0] [segmentTaskNum=32] [channelTaskNum=0] 2024-02-29 01:31:20.175 [2024/02/28 17:31:20.175 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=31] ["time cost"=1.564081ms] 2024-02-29 01:31:20.175 [2024/02/28 17:31:20.175 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136625286] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626526789] 2024-02-29 01:31:20.175 [2024/02/28 17:31:20.175 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=31] [committedNum=0] [toRemoveNum=1] 2024-02-29 01:31:20.175 [2024/02/28 17:31:20.175 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=31] [waitingTaskNum=0] [segmentTaskNum=31] [channelTaskNum=0] 2024-02-29 01:31:20.183 [2024/02/28 17:31:20.183 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626648] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.184 [2024/02/28 17:31:20.183 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626627] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.200 [2024/02/28 17:31:20.200 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626644] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.213 [2024/02/28 17:31:20.213 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:20.213 [2024/02/28 17:31:20.213 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=2c16d0b15754756fdafd46fdbc532ea7] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:20.230 [2024/02/28 17:31:20.230 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626620] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:20.233 [2024/02/28 17:31:20.233 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626620] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] [elapsed=3.072914ms] 2024-02-29 01:31:20.260 [2024/02/28 17:31:20.260 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626632] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.260 [2024/02/28 17:31:20.260 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626633] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.260 [2024/02/28 17:31:20.260 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626636] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.260 [2024/02/28 17:31:20.260 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626646] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.260 [2024/02/28 17:31:20.260 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626631] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.280 [2024/02/28 17:31:20.280 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626643] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.282 [2024/02/28 17:31:20.281 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626635] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.297 [2024/02/28 17:31:20.297 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626638] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.297 [2024/02/28 17:31:20.297 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626642] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.297 [2024/02/28 17:31:20.297 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626626] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.313 [2024/02/28 17:31:20.312 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:20.313 [2024/02/28 17:31:20.312 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=d016a1215e6b97f53482048e8ca02a0f] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:20.315 [2024/02/28 17:31:20.315 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:20.315 [2024/02/28 17:31:20.315 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=1830eb02d73de4892e4b87bee2d7d9e3] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:20.316 [2024/02/28 17:31:20.316 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626630] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.316 [2024/02/28 17:31:20.316 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626624] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.316 [2024/02/28 17:31:20.316 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626625] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.318 [2024/02/28 17:31:20.318 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=1bf53439894187540691bbc38d90ab2b] [collections="[448039877626298924]"] 2024-02-29 01:31:20.326 [2024/02/28 17:31:20.326 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:20.326 [2024/02/28 17:31:20.326 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=6bbcb321da24d9e8278b61008c3ff8ca] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:20.351 [2024/02/28 17:31:20.351 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626617] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:20.355 [2024/02/28 17:31:20.355 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626617] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] [elapsed=4.408713ms] 2024-02-29 01:31:20.386 [2024/02/28 17:31:20.385 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626645] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:20.424 [2024/02/28 17:31:20.424 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626523124] [error="node offline[node=8]"] 2024-02-29 01:31:20.424 [2024/02/28 17:31:20.424 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_1_448039877626298924v1 is not available in any replica"] [traceID=93a12f55554c7955ccd68c878ea593f4] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:20.444 [2024/02/28 17:31:20.443 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=0f59b4bbf11660a0fec8579eba201baf] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298931,448039877626298953,448039877626298959,448039877626298981,448039877626298970,448039877626299033,448039877626298993,448039877626298965,448039877626298987,448039877626299011,448039877626298936,448039877626298942,448039877626298999,448039877626299022,448039877626298976,448039877626299016,448039877626299027,448039877626299005,448039877626298948]"] 2024-02-29 01:31:20.444 [2024/02/28 17:31:20.444 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=0f59b4bbf11660a0fec8579eba201baf] [collectionID=448039877626298924] 2024-02-29 01:31:20.444 [2024/02/28 17:31:20.444 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=0f59b4bbf11660a0fec8579eba201baf] [collectionID=448039877626298924] 2024-02-29 01:31:20.444 [2024/02/28 17:31:20.444 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:20.444 [2024/02/28 17:31:20.444 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=0f59b4bbf11660a0fec8579eba201baf] [collectionID=448039877626298924] 2024-02-29 01:31:20.444 [2024/02/28 17:31:20.444 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=0f59b4bbf11660a0fec8579eba201baf] [collectionID=448039877626298924] 2024-02-29 01:31:20.451 [2024/02/28 17:31:20.451 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=5329b910b189a7bc1ed5453fe8480650] [collections="[448039877626298924]"] 2024-02-29 01:31:20.483 [2024/02/28 17:31:20.482 +00:00] [INFO] [task/executor.go:211] ["load segments done"] [taskID=1709136625287] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626552811] [node=3] [source=segment_checker] [shardLeader=3] [elapsed=36.839508493s] 2024-02-29 01:31:20.532 [2024/02/28 17:31:20.532 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626637] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:20.533 [2024/02/28 17:31:20.532 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626637] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:20.664 [2024/02/28 17:31:20.663 +00:00] [INFO] [task/scheduler.go:388] ["replace old task, the new one with higher priority"] [oldID=1709136625287] [oldPriority=Normal] [newID=0] [newPriority=High] 2024-02-29 01:31:20.664 [2024/02/28 17:31:20.664 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136625287] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626552811] 2024-02-29 01:31:20.664 [2024/02/28 17:31:20.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626649] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626552811][leader=3]"] 2024-02-29 01:31:20.664 [2024/02/28 17:31:20.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626650] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626524214][leader=3]"] 2024-02-29 01:31:20.664 [2024/02/28 17:31:20.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626651] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:20.664 [2024/02/28 17:31:20.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626652] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:20.664 [2024/02/28 17:31:20.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626653] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:20.664 [2024/02/28 17:31:20.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626654] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:20.665 [2024/02/28 17:31:20.664 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:20.665 [2024/02/28 17:31:20.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:20.665 [2024/02/28 17:31:20.665 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626527182] 2024-02-29 01:31:20.665 [2024/02/28 17:31:20.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626527182]"] 2024-02-29 01:31:20.668 [2024/02/28 17:31:20.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.668 [2024/02/28 17:31:20.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.668 [2024/02/28 17:31:20.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.668 [2024/02/28 17:31:20.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.668 [2024/02/28 17:31:20.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.668 [2024/02/28 17:31:20.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.668 [2024/02/28 17:31:20.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.668 [2024/02/28 17:31:20.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.668 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.668 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.668 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.668 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.668 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.668 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.668 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626655] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761400]"] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626656] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759971]"] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626657] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760570]"] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626658] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759849]"] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626659] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759777]"] 2024-02-29 01:31:20.669 [2024/02/28 17:31:20.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626660] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742592]"] 2024-02-29 01:31:20.700 [2024/02/28 17:31:20.700 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626654] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:20.700 [2024/02/28 17:31:20.700 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626654] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:20.700 [2024/02/28 17:31:20.700 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626654] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:20.700 [2024/02/28 17:31:20.700 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626649] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626552811] [taskType=Grow] 2024-02-29 01:31:20.700 [2024/02/28 17:31:20.700 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626649] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626552811]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626552811]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626552811]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:20.700 [2024/02/28 17:31:20.700 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626649] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626552811]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626552811]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626552811]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:20.700 [2024/02/28 17:31:20.700 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626653] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:20.700 [2024/02/28 17:31:20.700 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626653] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:20.700 [2024/02/28 17:31:20.700 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626653] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:20.700 [2024/02/28 17:31:20.700 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626651] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:20.700 [2024/02/28 17:31:20.700 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626651] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:20.700 [2024/02/28 17:31:20.700 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626651] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:20.700 [2024/02/28 17:31:20.700 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626652] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:20.700 [2024/02/28 17:31:20.700 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626652] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:20.700 [2024/02/28 17:31:20.700 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626652] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:20.701 [2024/02/28 17:31:20.701 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626654] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:20.701 [2024/02/28 17:31:20.701 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626649] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626552811] 2024-02-29 01:31:20.701 [2024/02/28 17:31:20.701 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626653] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:20.701 [2024/02/28 17:31:20.701 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626651] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:20.701 [2024/02/28 17:31:20.701 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626652] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:20.701 [2024/02/28 17:31:20.701 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=7] [toRemoveNum=5] 2024-02-29 01:31:20.701 [2024/02/28 17:31:20.701 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=37] [waitingTaskNum=0] [segmentTaskNum=37] [channelTaskNum=0] 2024-02-29 01:31:20.702 [2024/02/28 17:31:20.701 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626659] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:20.702 [2024/02/28 17:31:20.701 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626650] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:20.702 [2024/02/28 17:31:20.702 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626660] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:20.702 [2024/02/28 17:31:20.702 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626656] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:20.702 [2024/02/28 17:31:20.702 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626657] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:20.702 [2024/02/28 17:31:20.702 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626655] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:20.702 [2024/02/28 17:31:20.702 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626658] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:20.702 [2024/02/28 17:31:20.702 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=34] ["time cost"=316.867µs] 2024-02-29 01:31:20.702 [2024/02/28 17:31:20.702 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626620] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626527785] 2024-02-29 01:31:20.702 [2024/02/28 17:31:20.702 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626617] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626528386] 2024-02-29 01:31:20.702 [2024/02/28 17:31:20.702 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626637] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:20.702 [2024/02/28 17:31:20.702 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:20.702 [2024/02/28 17:31:20.702 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626637] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761572] 2024-02-29 01:31:20.702 [2024/02/28 17:31:20.702 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=34] [committedNum=7] [toRemoveNum=3] 2024-02-29 01:31:20.702 [2024/02/28 17:31:20.702 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=34] [waitingTaskNum=0] [segmentTaskNum=34] [channelTaskNum=0] 2024-02-29 01:31:20.844 [2024/02/28 17:31:20.844 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=5251ce2c0d012f595e1985382062bc82] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626299011,448039877626299027,448039877626299005,448039877626298942,448039877626298981,448039877626298936,448039877626299022,448039877626299016,448039877626298993,448039877626298987,448039877626299033,448039877626298948,448039877626298965,448039877626298999,448039877626298976,448039877626298931,448039877626298959,448039877626298953,448039877626298970]"] 2024-02-29 01:31:20.844 [2024/02/28 17:31:20.844 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=5251ce2c0d012f595e1985382062bc82] [collectionID=448039877626298924] 2024-02-29 01:31:20.844 [2024/02/28 17:31:20.844 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=5251ce2c0d012f595e1985382062bc82] [collectionID=448039877626298924] 2024-02-29 01:31:20.844 [2024/02/28 17:31:20.844 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:20.845 [2024/02/28 17:31:20.845 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=5251ce2c0d012f595e1985382062bc82] [collectionID=448039877626298924] 2024-02-29 01:31:20.845 [2024/02/28 17:31:20.845 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=5251ce2c0d012f595e1985382062bc82] [collectionID=448039877626298924] 2024-02-29 01:31:20.851 [2024/02/28 17:31:20.851 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=81106daf89598aa8d2bdad0626a83a49] [collections="[448039877626298924]"] 2024-02-29 01:31:20.855 [2024/02/28 17:31:20.854 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:20.855 [2024/02/28 17:31:20.855 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=6891499dd107ad7069ac5fe8d22b0a0d] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:21.048 [2024/02/28 17:31:21.048 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626657] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:21.068 [2024/02/28 17:31:21.068 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626659] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:21.068 [2024/02/28 17:31:21.068 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626660] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:21.093 [2024/02/28 17:31:21.090 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626656] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:21.093 [2024/02/28 17:31:21.090 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626655] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:21.093 [2024/02/28 17:31:21.091 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626658] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:21.112 [2024/02/28 17:31:21.112 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626650] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:21.124 [2024/02/28 17:31:21.123 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626641] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:21.124 [2024/02/28 17:31:21.123 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626641] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:21.141 [2024/02/28 17:31:21.141 +00:00] [WARN] [task/executor.go:468] ["failed to sync distribution"] [taskID=1709136626650] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:21.141 [2024/02/28 17:31:21.141 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626650] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:21.156 [2024/02/28 17:31:21.156 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=77455ca113916ce0fcfc9d3ca6caf10f] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298959,448039877626298993,448039877626299016,448039877626298999,448039877626299027,448039877626298970,448039877626298965,448039877626298942,448039877626299022,448039877626298948,448039877626298931,448039877626299011,448039877626299033,448039877626298976,448039877626298953,448039877626298987,448039877626299005,448039877626298981,448039877626298936]"] 2024-02-29 01:31:21.157 [2024/02/28 17:31:21.156 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=77455ca113916ce0fcfc9d3ca6caf10f] [collectionID=448039877626298924] 2024-02-29 01:31:21.157 [2024/02/28 17:31:21.156 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=77455ca113916ce0fcfc9d3ca6caf10f] [collectionID=448039877626298924] 2024-02-29 01:31:21.157 [2024/02/28 17:31:21.156 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:21.157 [2024/02/28 17:31:21.157 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=77455ca113916ce0fcfc9d3ca6caf10f] [collectionID=448039877626298924] 2024-02-29 01:31:21.157 [2024/02/28 17:31:21.157 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=77455ca113916ce0fcfc9d3ca6caf10f] [collectionID=448039877626298924] 2024-02-29 01:31:21.162 [2024/02/28 17:31:21.162 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=d22063b870645e0ff34dab82294ed93e] [collections="[448039877626298924]"] 2024-02-29 01:31:21.169 [2024/02/28 17:31:21.169 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:21.169 [2024/02/28 17:31:21.169 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=dd39e744f6e818e85d5e865eddb57508] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:21.169 [2024/02/28 17:31:21.169 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=34] [waitingTaskNum=0] [segmentTaskNum=34] [channelTaskNum=0] 2024-02-29 01:31:21.170 [2024/02/28 17:31:21.170 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=32] ["time cost"=200.12µs] 2024-02-29 01:31:21.170 [2024/02/28 17:31:21.170 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626650] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626524214] 2024-02-29 01:31:21.170 [2024/02/28 17:31:21.170 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626641] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:21.170 [2024/02/28 17:31:21.170 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:21.170 [2024/02/28 17:31:21.170 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626641] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761556] 2024-02-29 01:31:21.170 [2024/02/28 17:31:21.170 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=32] [committedNum=0] [toRemoveNum=2] 2024-02-29 01:31:21.170 [2024/02/28 17:31:21.170 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=32] [waitingTaskNum=0] [segmentTaskNum=32] [channelTaskNum=0] 2024-02-29 01:31:21.320 [2024/02/28 17:31:21.320 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626648] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:21.320 [2024/02/28 17:31:21.320 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626648] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:21.320 [2024/02/28 17:31:21.320 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626644] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:21.320 [2024/02/28 17:31:21.320 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626644] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:21.420 [2024/02/28 17:31:21.420 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626623] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:21.420 [2024/02/28 17:31:21.420 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626623] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:21.420 [2024/02/28 17:31:21.420 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626634] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:21.420 [2024/02/28 17:31:21.420 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626634] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:21.420 [2024/02/28 17:31:21.420 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626629] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:21.420 [2024/02/28 17:31:21.420 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626629] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:21.469 [2024/02/28 17:31:21.469 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626628] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:21.469 [2024/02/28 17:31:21.469 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626628] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:21.469 [2024/02/28 17:31:21.469 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626631] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:21.469 [2024/02/28 17:31:21.469 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626631] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:21.469 [2024/02/28 17:31:21.469 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626627] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:21.469 [2024/02/28 17:31:21.469 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626627] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:21.469 [2024/02/28 17:31:21.469 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626640] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:21.470 [2024/02/28 17:31:21.469 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626640] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:21.470 [2024/02/28 17:31:21.469 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626639] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:21.470 [2024/02/28 17:31:21.469 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626639] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:21.470 [2024/02/28 17:31:21.469 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626636] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:21.470 [2024/02/28 17:31:21.469 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626636] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:21.470 [2024/02/28 17:31:21.469 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626633] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:21.470 [2024/02/28 17:31:21.469 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626633] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:21.470 [2024/02/28 17:31:21.469 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626647] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:21.470 [2024/02/28 17:31:21.469 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626632] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:21.470 [2024/02/28 17:31:21.469 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626632] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:21.470 [2024/02/28 17:31:21.469 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626647] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:21.496 [2024/02/28 17:31:21.496 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=1d96ad4267f441f02b0188ea4a25a914] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298942,448039877626298948,448039877626299033,448039877626298987,448039877626298999,448039877626298970,448039877626298965,448039877626299005,448039877626298936,448039877626299016,448039877626298953,448039877626299027,448039877626298959,448039877626299011,448039877626298976,448039877626298981,448039877626298931,448039877626298993,448039877626299022]"] 2024-02-29 01:31:21.497 [2024/02/28 17:31:21.496 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=1d96ad4267f441f02b0188ea4a25a914] [collectionID=448039877626298924] 2024-02-29 01:31:21.497 [2024/02/28 17:31:21.496 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=1d96ad4267f441f02b0188ea4a25a914] [collectionID=448039877626298924] 2024-02-29 01:31:21.497 [2024/02/28 17:31:21.497 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:21.498 [2024/02/28 17:31:21.497 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=1d96ad4267f441f02b0188ea4a25a914] [collectionID=448039877626298924] 2024-02-29 01:31:21.498 [2024/02/28 17:31:21.497 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=1d96ad4267f441f02b0188ea4a25a914] [collectionID=448039877626298924] 2024-02-29 01:31:21.502 [2024/02/28 17:31:21.502 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=17c5ed74d0422baa8992889a7a4b5335] [collections="[448039877626298924]"] 2024-02-29 01:31:21.515 [2024/02/28 17:31:21.515 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626643] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:21.515 [2024/02/28 17:31:21.515 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626643] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:21.515 [2024/02/28 17:31:21.515 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626638] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:21.515 [2024/02/28 17:31:21.515 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626638] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:21.515 [2024/02/28 17:31:21.515 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626642] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:21.515 [2024/02/28 17:31:21.515 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626642] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:21.614 [2024/02/28 17:31:21.614 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626646] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:21.614 [2024/02/28 17:31:21.614 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626646] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:21.663 [2024/02/28 17:31:21.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626661] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626524214][leader=3]"] 2024-02-29 01:31:21.663 [2024/02/28 17:31:21.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626662] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:21.663 [2024/02/28 17:31:21.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626663] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:21.663 [2024/02/28 17:31:21.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626664] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626527785][leader=3]"] 2024-02-29 01:31:21.663 [2024/02/28 17:31:21.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626665] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:21.663 [2024/02/28 17:31:21.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626666] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:21.663 [2024/02/28 17:31:21.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626667] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626528386][leader=3]"] 2024-02-29 01:31:21.664 [2024/02/28 17:31:21.664 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626523124] [error="node offline[node=8]"] 2024-02-29 01:31:21.664 [2024/02/28 17:31:21.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:21.664 [2024/02/28 17:31:21.664 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:21.664 [2024/02/28 17:31:21.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:21.665 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:21.666 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:21.666 [2024/02/28 17:31:21.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:21.666 [2024/02/28 17:31:21.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626668] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761572]"] 2024-02-29 01:31:21.666 [2024/02/28 17:31:21.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626669] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761556]"] 2024-02-29 01:31:21.669 [2024/02/28 17:31:21.669 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626662] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.669 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626662] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626662] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.670 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626666] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.670 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626666] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626666] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.670 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626665] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.670 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626665] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626665] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.670 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626663] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.670 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626663] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626663] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.670 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626662] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.670 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626666] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.670 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626665] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.670 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626663] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.670 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=5] [toRemoveNum=4] 2024-02-29 01:31:21.670 [2024/02/28 17:31:21.670 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=37] [waitingTaskNum=0] [segmentTaskNum=37] [channelTaskNum=0] 2024-02-29 01:31:21.672 [2024/02/28 17:31:21.671 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626664] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:21.672 [2024/02/28 17:31:21.671 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626668] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:21.672 [2024/02/28 17:31:21.671 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=19] ["time cost"=502.056µs] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.671 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626643] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.671 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.671 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626667] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626643] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759770] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.671 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626639] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.671 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626639] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759826] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.672 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626633] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.672 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.672 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626633] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760748] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.672 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626629] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.672 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.672 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626629] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759628] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.672 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626632] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.672 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.672 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626632] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761308] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.672 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626648] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.672 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.672 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626648] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742556] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.671 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626661] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.671 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626669] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.672 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626646] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.672 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:21.673 [2024/02/28 17:31:21.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626646] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759968] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626642] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626642] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759612] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626628] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626628] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761482] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626627] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626627] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760744] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626636] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626636] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760422] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626644] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626644] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760215] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626623] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626623] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760025] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626640] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:21.674 [2024/02/28 17:31:21.673 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:21.675 [2024/02/28 17:31:21.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626640] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760696] 2024-02-29 01:31:21.675 [2024/02/28 17:31:21.673 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626647] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:21.675 [2024/02/28 17:31:21.673 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:21.675 [2024/02/28 17:31:21.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626647] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760374] 2024-02-29 01:31:21.675 [2024/02/28 17:31:21.674 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626638] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:21.675 [2024/02/28 17:31:21.674 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:21.675 [2024/02/28 17:31:21.674 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626638] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760192] 2024-02-29 01:31:21.675 [2024/02/28 17:31:21.674 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626631] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:21.675 [2024/02/28 17:31:21.674 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:21.675 [2024/02/28 17:31:21.674 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626631] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760970] 2024-02-29 01:31:21.675 [2024/02/28 17:31:21.674 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626634] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:21.675 [2024/02/28 17:31:21.674 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:21.675 [2024/02/28 17:31:21.674 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626634] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760366] 2024-02-29 01:31:21.675 [2024/02/28 17:31:21.674 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=19] [committedNum=5] [toRemoveNum=18] 2024-02-29 01:31:21.675 [2024/02/28 17:31:21.674 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=19] [waitingTaskNum=0] [segmentTaskNum=19] [channelTaskNum=0] 2024-02-29 01:31:21.710 [2024/02/28 17:31:21.710 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626397] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626548132] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626548132/448039877626558600]"] 2024-02-29 01:31:21.710 [2024/02/28 17:31:21.710 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626635] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:21.710 [2024/02/28 17:31:21.710 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626397] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626548132/448039877626558600]"] 2024-02-29 01:31:21.710 [2024/02/28 17:31:21.710 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626635] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:21.710 [2024/02/28 17:31:21.710 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626626] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:21.710 [2024/02/28 17:31:21.710 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626626] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:21.747 [2024/02/28 17:31:21.747 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=776e27eaa971c7f2c01c8b507f56d115] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298993,448039877626298981,448039877626299027,448039877626299033,448039877626298965,448039877626299005,448039877626298976,448039877626298936,448039877626299011,448039877626298970,448039877626298959,448039877626298953,448039877626298948,448039877626298942,448039877626298987,448039877626299016,448039877626299022,448039877626298999,448039877626298931]"] 2024-02-29 01:31:21.747 [2024/02/28 17:31:21.747 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=776e27eaa971c7f2c01c8b507f56d115] [collectionID=448039877626298924] 2024-02-29 01:31:21.747 [2024/02/28 17:31:21.747 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=776e27eaa971c7f2c01c8b507f56d115] [collectionID=448039877626298924] 2024-02-29 01:31:21.747 [2024/02/28 17:31:21.747 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:21.748 [2024/02/28 17:31:21.748 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=776e27eaa971c7f2c01c8b507f56d115] [collectionID=448039877626298924] 2024-02-29 01:31:21.748 [2024/02/28 17:31:21.748 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=776e27eaa971c7f2c01c8b507f56d115] [collectionID=448039877626298924] 2024-02-29 01:31:21.753 [2024/02/28 17:31:21.753 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=83dc01d24afac07c02ede748c5c2a017] [collections="[448039877626298924]"] 2024-02-29 01:31:21.762 [2024/02/28 17:31:21.762 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626659] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:21.763 [2024/02/28 17:31:21.762 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626659] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:21.763 [2024/02/28 17:31:21.762 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626657] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:21.763 [2024/02/28 17:31:21.762 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626625] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:21.763 [2024/02/28 17:31:21.762 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626660] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:21.763 [2024/02/28 17:31:21.763 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626625] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:21.763 [2024/02/28 17:31:21.763 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626660] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:21.763 [2024/02/28 17:31:21.762 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626657] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:21.763 [2024/02/28 17:31:21.763 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626655] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:21.763 [2024/02/28 17:31:21.763 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626655] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:21.763 [2024/02/28 17:31:21.763 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626645] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:21.763 [2024/02/28 17:31:21.763 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626624] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:21.763 [2024/02/28 17:31:21.763 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626645] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:21.763 [2024/02/28 17:31:21.763 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626624] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:21.763 [2024/02/28 17:31:21.763 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626630] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:21.763 [2024/02/28 17:31:21.763 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626630] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:21.788 [2024/02/28 17:31:21.788 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626656] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:21.788 [2024/02/28 17:31:21.788 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626656] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:21.829 [2024/02/28 17:31:21.829 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626658] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:21.829 [2024/02/28 17:31:21.829 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626658] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:22.058 [2024/02/28 17:31:22.058 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626669] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:22.058 [2024/02/28 17:31:22.058 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626668] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:22.105 [2024/02/28 17:31:22.105 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626667] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:22.108 [2024/02/28 17:31:22.108 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626667] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] [elapsed=2.911552ms] 2024-02-29 01:31:22.125 [2024/02/28 17:31:22.125 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626661] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:22.150 [2024/02/28 17:31:22.150 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626664] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:22.150 [2024/02/28 17:31:22.150 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=40f83624b92cbbccb356508a0d2bb3a7] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298976,448039877626299033,448039877626299005,448039877626298993,448039877626299011,448039877626298987,448039877626298970,448039877626298959,448039877626299022,448039877626298981,448039877626299027,448039877626298953,448039877626299016,448039877626298931,448039877626298948,448039877626298942,448039877626298999,448039877626298936,448039877626298965]"] 2024-02-29 01:31:22.151 [2024/02/28 17:31:22.150 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=40f83624b92cbbccb356508a0d2bb3a7] [collectionID=448039877626298924] 2024-02-29 01:31:22.151 [2024/02/28 17:31:22.150 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=40f83624b92cbbccb356508a0d2bb3a7] [collectionID=448039877626298924] 2024-02-29 01:31:22.151 [2024/02/28 17:31:22.151 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:22.151 [2024/02/28 17:31:22.151 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=40f83624b92cbbccb356508a0d2bb3a7] [collectionID=448039877626298924] 2024-02-29 01:31:22.151 [2024/02/28 17:31:22.151 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=40f83624b92cbbccb356508a0d2bb3a7] [collectionID=448039877626298924] 2024-02-29 01:31:22.154 [2024/02/28 17:31:22.154 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626664] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] [elapsed=4.063699ms] 2024-02-29 01:31:22.156 [2024/02/28 17:31:22.156 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=8c9f1513ca008df7eb7095800016ddb8] [collections="[448039877626298924]"] 2024-02-29 01:31:22.158 [2024/02/28 17:31:22.157 +00:00] [WARN] [task/executor.go:468] ["failed to sync distribution"] [taskID=1709136626661] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:22.158 [2024/02/28 17:31:22.157 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626661] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:22.171 [2024/02/28 17:31:22.171 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626669] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:22.171 [2024/02/28 17:31:22.171 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626669] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:22.172 [2024/02/28 17:31:22.171 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=19] [waitingTaskNum=0] [segmentTaskNum=19] [channelTaskNum=0] 2024-02-29 01:31:22.172 [2024/02/28 17:31:22.172 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=2] ["time cost"=170.312µs] 2024-02-29 01:31:22.172 [2024/02/28 17:31:22.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626667] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626528386] 2024-02-29 01:31:22.172 [2024/02/28 17:31:22.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626664] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626527785] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626661] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626524214] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626669] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626669] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761556] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626655] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626655] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761400] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626630] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.172 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626630] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761031] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.172 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626397] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626548132] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626548132/448039877626558600]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.172 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626548132/448039877626558600]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626397] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626548132] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626626] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626626] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759725] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626645] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626645] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760199] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626624] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626624] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760757] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626657] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626657] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760570] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626635] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626635] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761143] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626659] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:22.173 [2024/02/28 17:31:22.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:22.174 [2024/02/28 17:31:22.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626659] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759777] 2024-02-29 01:31:22.174 [2024/02/28 17:31:22.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626656] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:22.174 [2024/02/28 17:31:22.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:22.174 [2024/02/28 17:31:22.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626656] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759971] 2024-02-29 01:31:22.174 [2024/02/28 17:31:22.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626625] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:22.174 [2024/02/28 17:31:22.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:22.174 [2024/02/28 17:31:22.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626625] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760430] 2024-02-29 01:31:22.174 [2024/02/28 17:31:22.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626660] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:22.174 [2024/02/28 17:31:22.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:22.174 [2024/02/28 17:31:22.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626660] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742592] 2024-02-29 01:31:22.174 [2024/02/28 17:31:22.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626658] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:22.174 [2024/02/28 17:31:22.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:22.174 [2024/02/28 17:31:22.173 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626658] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759849] 2024-02-29 01:31:22.174 [2024/02/28 17:31:22.173 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=2] [committedNum=0] [toRemoveNum=17] 2024-02-29 01:31:22.174 [2024/02/28 17:31:22.173 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=2] [waitingTaskNum=0] [segmentTaskNum=2] [channelTaskNum=0] 2024-02-29 01:31:22.291 [2024/02/28 17:31:22.290 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=19615326ca8e05d518fd0492e52f81da] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298993,448039877626298959,448039877626298981,448039877626299005,448039877626299022,448039877626298931,448039877626298999,448039877626298970,448039877626299027,448039877626298976,448039877626299011,448039877626298942,448039877626298987,448039877626299016,448039877626298948,448039877626298965,448039877626298953,448039877626298936,448039877626299033]"] 2024-02-29 01:31:22.292 [2024/02/28 17:31:22.291 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=19615326ca8e05d518fd0492e52f81da] [collectionID=448039877626298924] 2024-02-29 01:31:22.292 [2024/02/28 17:31:22.291 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=19615326ca8e05d518fd0492e52f81da] [collectionID=448039877626298924] 2024-02-29 01:31:22.292 [2024/02/28 17:31:22.291 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:22.293 [2024/02/28 17:31:22.292 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=19615326ca8e05d518fd0492e52f81da] [collectionID=448039877626298924] 2024-02-29 01:31:22.293 [2024/02/28 17:31:22.292 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=19615326ca8e05d518fd0492e52f81da] [collectionID=448039877626298924] 2024-02-29 01:31:22.298 [2024/02/28 17:31:22.298 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=740775bdac4368ec1420ed7efe35ea75] [collections="[448039877626298924]"] 2024-02-29 01:31:22.303 [2024/02/28 17:31:22.303 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:22.303 [2024/02/28 17:31:22.303 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=c83dd58238c37bdc7a25e5f5ed00a59f] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:22.309 [2024/02/28 17:31:22.309 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626668] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:22.309 [2024/02/28 17:31:22.309 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626668] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:22.488 [2024/02/28 17:31:22.487 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:22.488 [2024/02/28 17:31:22.487 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=2e96989fb831216a89cb13844713019b] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:22.582 [2024/02/28 17:31:22.581 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=f2f1164e484f5961b3909d00f0b1f88a] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298970,448039877626298942,448039877626299011,448039877626298948,448039877626299027,448039877626298931,448039877626298999,448039877626299016,448039877626299033,448039877626298936,448039877626298987,448039877626298976,448039877626298953,448039877626298959,448039877626299005,448039877626298993,448039877626298965,448039877626299022,448039877626298981]"] 2024-02-29 01:31:22.582 [2024/02/28 17:31:22.581 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=f2f1164e484f5961b3909d00f0b1f88a] [collectionID=448039877626298924] 2024-02-29 01:31:22.582 [2024/02/28 17:31:22.581 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=f2f1164e484f5961b3909d00f0b1f88a] [collectionID=448039877626298924] 2024-02-29 01:31:22.582 [2024/02/28 17:31:22.581 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:22.583 [2024/02/28 17:31:22.583 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=f2f1164e484f5961b3909d00f0b1f88a] [collectionID=448039877626298924] 2024-02-29 01:31:22.583 [2024/02/28 17:31:22.583 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=f2f1164e484f5961b3909d00f0b1f88a] [collectionID=448039877626298924] 2024-02-29 01:31:22.588 [2024/02/28 17:31:22.587 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=027ed135a2df81e16746929b24e72828] [collections="[448039877626298924]"] 2024-02-29 01:31:22.663 [2024/02/28 17:31:22.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626670] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:22.663 [2024/02/28 17:31:22.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626671] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:22.663 [2024/02/28 17:31:22.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626672] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626528386][leader=3]"] 2024-02-29 01:31:22.663 [2024/02/28 17:31:22.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626673] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626524214][leader=3]"] 2024-02-29 01:31:22.664 [2024/02/28 17:31:22.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626674] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626527785][leader=3]"] 2024-02-29 01:31:22.664 [2024/02/28 17:31:22.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626675] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:22.664 [2024/02/28 17:31:22.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626676] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:22.664 [2024/02/28 17:31:22.664 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:22.664 [2024/02/28 17:31:22.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:22.664 [2024/02/28 17:31:22.664 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:22.664 [2024/02/28 17:31:22.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.665 [2024/02/28 17:31:22.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626677] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877626548132]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626678] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742556]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626679] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742592]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626680] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761400]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626681] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760025]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626682] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760757]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626683] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760430]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626684] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759725]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626685] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759971]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626686] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760744]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626687] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760570]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626688] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761482]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626689] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759628]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626690] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761031]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626691] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760970]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626692] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761308]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626693] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760366]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626694] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760748]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626695] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760192]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626696] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761143]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626697] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760422]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626698] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760696]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626699] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761556]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626700] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759826]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626701] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759612]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626702] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759770]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626703] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759849]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626704] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759777]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626705] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760199]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626706] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760215]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626707] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759968]"] 2024-02-29 01:31:22.666 [2024/02/28 17:31:22.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626708] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760374]"] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.671 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626675] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.671 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626675] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.671 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626675] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.671 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626670] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.672 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626670] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.672 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626670] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.672 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626676] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.672 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626676] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.672 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626676] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.672 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626671] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.672 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626671] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.672 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626671] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626675] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626670] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626676] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.674 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626671] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.674 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=35] [toRemoveNum=4] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.674 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=37] [waitingTaskNum=0] [segmentTaskNum=37] [channelTaskNum=0] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626674] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626677] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626697] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626691] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626673] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626693] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626682] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626694] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626672] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626692] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626679] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.676 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626704] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.689 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626698] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.689 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626690] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.690 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626701] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.690 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626678] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.690 [2024/02/28 17:31:22.689 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626683] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.690 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626705] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.690 [2024/02/28 17:31:22.689 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626688] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.690 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626696] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.690 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626695] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.690 [2024/02/28 17:31:22.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626680] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.690 [2024/02/28 17:31:22.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626684] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.690 [2024/02/28 17:31:22.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626703] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.690 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626708] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.691 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626687] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.691 [2024/02/28 17:31:22.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626702] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.691 [2024/02/28 17:31:22.691 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626707] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.691 [2024/02/28 17:31:22.691 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626681] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.691 [2024/02/28 17:31:22.691 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626689] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.691 [2024/02/28 17:31:22.691 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626686] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.691 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626700] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.691 [2024/02/28 17:31:22.691 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626685] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.692 [2024/02/28 17:31:22.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626699] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.693 [2024/02/28 17:31:22.692 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=36] ["time cost"=17.279137ms] 2024-02-29 01:31:22.693 [2024/02/28 17:31:22.692 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626668] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:22.693 [2024/02/28 17:31:22.692 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:22.693 [2024/02/28 17:31:22.692 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626668] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761572] 2024-02-29 01:31:22.693 [2024/02/28 17:31:22.692 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=36] [committedNum=35] [toRemoveNum=1] 2024-02-29 01:31:22.693 [2024/02/28 17:31:22.692 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=36] [waitingTaskNum=0] [segmentTaskNum=36] [channelTaskNum=0] 2024-02-29 01:31:22.693 [2024/02/28 17:31:22.692 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626706] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:22.716 [2024/02/28 17:31:22.716 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626527182] 2024-02-29 01:31:22.716 [2024/02/28 17:31:22.716 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=e5d1018c16bca6da6df9856d9733befa] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626527182]"] 2024-02-29 01:31:22.723 [2024/02/28 17:31:22.723 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:22.723 [2024/02/28 17:31:22.723 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_1_448039877626298924v1 is not available in any replica"] [traceID=bcec63f98745de906027f34632887fe1] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:23.172 [2024/02/28 17:31:23.171 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=36] [waitingTaskNum=0] [segmentTaskNum=36] [channelTaskNum=0] 2024-02-29 01:31:23.173 [2024/02/28 17:31:23.173 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=36] ["time cost"=325.114µs] 2024-02-29 01:31:23.173 [2024/02/28 17:31:23.173 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=36] [committedNum=0] [toRemoveNum=0] 2024-02-29 01:31:23.173 [2024/02/28 17:31:23.173 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=36] [waitingTaskNum=0] [segmentTaskNum=36] [channelTaskNum=0] 2024-02-29 01:31:23.231 [2024/02/28 17:31:23.231 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626684] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.231 [2024/02/28 17:31:23.231 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626694] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.231 [2024/02/28 17:31:23.231 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626695] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.231 [2024/02/28 17:31:23.231 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626696] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.232 [2024/02/28 17:31:23.232 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626689] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.233 [2024/02/28 17:31:23.233 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626705] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.234 [2024/02/28 17:31:23.233 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626673] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:23.262 [2024/02/28 17:31:23.262 +00:00] [WARN] [task/executor.go:468] ["failed to sync distribution"] [taskID=1709136626673] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:23.262 [2024/02/28 17:31:23.262 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626673] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:23.307 [2024/02/28 17:31:23.307 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626674] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:23.309 [2024/02/28 17:31:23.309 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626674] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] [elapsed=2.549502ms] 2024-02-29 01:31:23.420 [2024/02/28 17:31:23.420 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626707] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.421 [2024/02/28 17:31:23.420 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626693] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.421 [2024/02/28 17:31:23.421 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626683] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.421 [2024/02/28 17:31:23.421 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626677] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626548132] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.422 [2024/02/28 17:31:23.421 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626691] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.422 [2024/02/28 17:31:23.422 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=c84872e20ead5b79ac6aec0fd1b9a4d8] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298993,448039877626298948,448039877626298981,448039877626298942,448039877626298987,448039877626298999,448039877626299011,448039877626298959,448039877626298965,448039877626298976,448039877626298936,448039877626298931,448039877626299033,448039877626298953,448039877626299027,448039877626299005,448039877626299016,448039877626298970,448039877626299022]"] 2024-02-29 01:31:23.422 [2024/02/28 17:31:23.422 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=c84872e20ead5b79ac6aec0fd1b9a4d8] [collectionID=448039877626298924] 2024-02-29 01:31:23.422 [2024/02/28 17:31:23.422 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=c84872e20ead5b79ac6aec0fd1b9a4d8] [collectionID=448039877626298924] 2024-02-29 01:31:23.422 [2024/02/28 17:31:23.422 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:23.423 [2024/02/28 17:31:23.423 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=c84872e20ead5b79ac6aec0fd1b9a4d8] [collectionID=448039877626298924] 2024-02-29 01:31:23.423 [2024/02/28 17:31:23.423 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=c84872e20ead5b79ac6aec0fd1b9a4d8] [collectionID=448039877626298924] 2024-02-29 01:31:23.435 [2024/02/28 17:31:23.435 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=1c60080c083424151158e329aaa1ed97] [collections="[448039877626298924]"] 2024-02-29 01:31:23.499 [2024/02/28 17:31:23.499 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626672] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:23.502 [2024/02/28 17:31:23.502 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626672] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] [elapsed=2.901845ms] 2024-02-29 01:31:23.522 [2024/02/28 17:31:23.522 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626703] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.522 [2024/02/28 17:31:23.522 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626702] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.529 [2024/02/28 17:31:23.529 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626695] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:23.529 [2024/02/28 17:31:23.529 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626695] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:23.529 [2024/02/28 17:31:23.529 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626694] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:23.529 [2024/02/28 17:31:23.529 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626694] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:23.537 [2024/02/28 17:31:23.537 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626706] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.537 [2024/02/28 17:31:23.537 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626686] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.537 [2024/02/28 17:31:23.537 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626704] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.537 [2024/02/28 17:31:23.537 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626685] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.556 [2024/02/28 17:31:23.556 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:23.556 [2024/02/28 17:31:23.556 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=2db18c9b67d207c128b41d5779dba92a] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:23.557 [2024/02/28 17:31:23.557 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626679] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.574 [2024/02/28 17:31:23.574 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626699] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.591 [2024/02/28 17:31:23.590 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626681] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.644 [2024/02/28 17:31:23.644 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626687] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.644 [2024/02/28 17:31:23.644 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626682] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.644 [2024/02/28 17:31:23.644 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626697] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.665 [2024/02/28 17:31:23.664 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:23.665 [2024/02/28 17:31:23.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:23.665 [2024/02/28 17:31:23.664 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:23.665 [2024/02/28 17:31:23.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:23.665 [2024/02/28 17:31:23.665 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626709] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:23.665 [2024/02/28 17:31:23.665 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626710] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:23.665 [2024/02/28 17:31:23.665 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626711] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:23.665 [2024/02/28 17:31:23.665 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626712] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.666 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.667 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.667 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.667 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.667 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.667 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.667 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.667 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.667 [2024/02/28 17:31:23.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.667 [2024/02/28 17:31:23.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.667 [2024/02/28 17:31:23.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.667 [2024/02/28 17:31:23.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:23.667 [2024/02/28 17:31:23.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:23.667 [2024/02/28 17:31:23.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:23.667 [2024/02/28 17:31:23.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:23.667 [2024/02/28 17:31:23.667 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:23.669 [2024/02/28 17:31:23.668 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626713] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761572]"] 2024-02-29 01:31:23.676 [2024/02/28 17:31:23.674 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626696] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:23.676 [2024/02/28 17:31:23.675 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626696] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:23.676 [2024/02/28 17:31:23.675 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626710] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:23.676 [2024/02/28 17:31:23.675 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626710] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:23.676 [2024/02/28 17:31:23.675 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626705] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:23.676 [2024/02/28 17:31:23.675 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626710] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:23.676 [2024/02/28 17:31:23.675 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626709] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:23.676 [2024/02/28 17:31:23.675 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626705] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:23.676 [2024/02/28 17:31:23.675 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626709] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:23.676 [2024/02/28 17:31:23.676 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626709] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:23.676 [2024/02/28 17:31:23.676 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626711] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:23.676 [2024/02/28 17:31:23.676 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626711] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:23.676 [2024/02/28 17:31:23.676 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626711] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:23.676 [2024/02/28 17:31:23.676 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626712] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:23.676 [2024/02/28 17:31:23.676 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626712] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:23.677 [2024/02/28 17:31:23.676 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626712] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626710] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626709] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.676 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626711] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.677 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626712] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.677 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=634f73c0fede46e413fe096cd8bf2af2] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.677 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=1] [toRemoveNum=4] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.677 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=37] [waitingTaskNum=0] [segmentTaskNum=37] [channelTaskNum=0] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.678 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626713] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.678 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=30] ["time cost"=628.514µs] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.679 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626673] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626524214] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.679 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626672] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626528386] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.679 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626674] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626527785] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.679 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626694] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.679 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.679 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626694] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760748] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.679 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626705] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.679 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.679 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626705] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760199] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.679 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626696] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.679 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.679 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626696] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761143] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.679 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626695] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.679 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:23.679 [2024/02/28 17:31:23.679 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626695] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760192] 2024-02-29 01:31:23.680 [2024/02/28 17:31:23.679 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=30] [committedNum=1] [toRemoveNum=7] 2024-02-29 01:31:23.680 [2024/02/28 17:31:23.679 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=30] [waitingTaskNum=0] [segmentTaskNum=30] [channelTaskNum=0] 2024-02-29 01:31:23.680 [2024/02/28 17:31:23.680 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:23.680 [2024/02/28 17:31:23.680 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=93d6ee3a6db49cce27ae74f72bd4ca23] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:23.702 [2024/02/28 17:31:23.702 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626678] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.702 [2024/02/28 17:31:23.702 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626708] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.724 [2024/02/28 17:31:23.723 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626680] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.752 [2024/02/28 17:31:23.752 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626701] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.752 [2024/02/28 17:31:23.752 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626690] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.752 [2024/02/28 17:31:23.752 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626692] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.752 [2024/02/28 17:31:23.752 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626698] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.783 [2024/02/28 17:31:23.782 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:23.783 [2024/02/28 17:31:23.782 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=0f1383e54da18ec1172fddd42e4d3498] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:23.792 [2024/02/28 17:31:23.791 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626688] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.847 [2024/02/28 17:31:23.847 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626700] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:23.852 [2024/02/28 17:31:23.852 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:23.852 [2024/02/28 17:31:23.852 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=f59090ecbcbb7ed4f3f26af2168bc35b] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:23.910 [2024/02/28 17:31:23.909 +00:00] [INFO] [querycoordv2/services.go:838] ["get shard leaders request received"] [traceID=150a3b86de0e8514a70afce27427e097] [collectionID=448039877626298924] 2024-02-29 01:31:23.910 [2024/02/28 17:31:23.910 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:23.910 [2024/02/28 17:31:23.910 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=150a3b86de0e8514a70afce27427e097] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:23.911 [2024/02/28 17:31:23.911 +00:00] [DEBUG] [config/etcd_source.go:143] ["etcd refreshConfigurations"] [prefix=by-dev/config] [endpoints="[inverted-corn-136000-8-75-4542-etcd:2379]"] 2024-02-29 01:31:23.916 [2024/02/28 17:31:23.916 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:23.916 [2024/02/28 17:31:23.916 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=ac508bd7aff6d03dd329b79dcfc6b6e2] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:23.954 [2024/02/28 17:31:23.954 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626684] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:23.954 [2024/02/28 17:31:23.954 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626684] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:24.092 [2024/02/28 17:31:24.092 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626713] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:24.092 [2024/02/28 17:31:24.092 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:24.092 [2024/02/28 17:31:24.092 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=42959b5df9f38c15dca117e3553e3da8] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:24.093 [2024/02/28 17:31:24.092 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:24.093 [2024/02/28 17:31:24.092 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=933af00005c7105ee0791e02d1a11aa9] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:24.176 [2024/02/28 17:31:24.176 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:24.177 [2024/02/28 17:31:24.176 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=145d41ebcf02bb77f141f2ba8ef27b96] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:24.177 [2024/02/28 17:31:24.177 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=30] [waitingTaskNum=0] [segmentTaskNum=30] [channelTaskNum=0] 2024-02-29 01:31:24.178 [2024/02/28 17:31:24.178 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=29] ["time cost"=362.859µs] 2024-02-29 01:31:24.179 [2024/02/28 17:31:24.178 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626684] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:24.179 [2024/02/28 17:31:24.178 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:24.179 [2024/02/28 17:31:24.178 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626684] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759725] 2024-02-29 01:31:24.179 [2024/02/28 17:31:24.178 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=29] [committedNum=0] [toRemoveNum=1] 2024-02-29 01:31:24.179 [2024/02/28 17:31:24.178 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=29] [waitingTaskNum=0] [segmentTaskNum=29] [channelTaskNum=0] 2024-02-29 01:31:24.180 [2024/02/28 17:31:24.180 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626523124] [error="node offline[node=8]"] 2024-02-29 01:31:24.180 [2024/02/28 17:31:24.180 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_1_448039877626298924v1 is not available in any replica"] [traceID=fa3871845002924e5e13cdf53dd260bd] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:24.201 [2024/02/28 17:31:24.201 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626704] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:24.201 [2024/02/28 17:31:24.201 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626685] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:24.201 [2024/02/28 17:31:24.201 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626704] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:24.201 [2024/02/28 17:31:24.201 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626685] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:24.201 [2024/02/28 17:31:24.201 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626686] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:24.201 [2024/02/28 17:31:24.201 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626702] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:24.201 [2024/02/28 17:31:24.201 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626686] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:24.201 [2024/02/28 17:31:24.201 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626702] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:24.237 [2024/02/28 17:31:24.237 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:24.237 [2024/02/28 17:31:24.237 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=9afddaa49e0fcb87aa4b2892dea7d9b7] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:24.250 [2024/02/28 17:31:24.250 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626693] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:24.250 [2024/02/28 17:31:24.250 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626693] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:24.250 [2024/02/28 17:31:24.250 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626689] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:24.250 [2024/02/28 17:31:24.250 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626689] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:24.291 [2024/02/28 17:31:24.291 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:24.291 [2024/02/28 17:31:24.291 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=e64e60653672ec7a473a6e40e1783fa6] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:24.338 [2024/02/28 17:31:24.338 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626707] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:24.338 [2024/02/28 17:31:24.338 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626707] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:24.346 [2024/02/28 17:31:24.346 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:24.346 [2024/02/28 17:31:24.346 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=0cf7975da5db25b312f60e565ed6dd0e] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:24.347 [2024/02/28 17:31:24.347 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:24.347 [2024/02/28 17:31:24.347 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=f4d9048670cee8ae0c2ee9adbe6cbe47] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:24.350 [2024/02/28 17:31:24.350 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:24.350 [2024/02/28 17:31:24.350 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=d6febea60e5d613128206c7de728d884] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:24.351 [2024/02/28 17:31:24.351 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:24.351 [2024/02/28 17:31:24.351 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=c0b2b1cefddd5586f665932ab39547e4] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:24.351 [2024/02/28 17:31:24.351 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:24.351 [2024/02/28 17:31:24.351 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=8b92bddff152a4897b587a80fe927088] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:24.392 [2024/02/28 17:31:24.391 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=3d23f30aa599c0d25a737fd2391c9bdd] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298953,448039877626299011,448039877626298959,448039877626299005,448039877626299022,448039877626299027,448039877626298948,448039877626298993,448039877626298942,448039877626298987,448039877626298970,448039877626299016,448039877626298999,448039877626299033,448039877626298976,448039877626298965,448039877626298931,448039877626298981,448039877626298936]"] 2024-02-29 01:31:24.392 [2024/02/28 17:31:24.392 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=3d23f30aa599c0d25a737fd2391c9bdd] [collectionID=448039877626298924] 2024-02-29 01:31:24.392 [2024/02/28 17:31:24.392 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=3d23f30aa599c0d25a737fd2391c9bdd] [collectionID=448039877626298924] 2024-02-29 01:31:24.392 [2024/02/28 17:31:24.392 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:24.393 [2024/02/28 17:31:24.393 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=3d23f30aa599c0d25a737fd2391c9bdd] [collectionID=448039877626298924] 2024-02-29 01:31:24.393 [2024/02/28 17:31:24.393 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=3d23f30aa599c0d25a737fd2391c9bdd] [collectionID=448039877626298924] 2024-02-29 01:31:24.410 [2024/02/28 17:31:24.410 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:24.410 [2024/02/28 17:31:24.410 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=64cb6568688c90ebf28eb984ae39e1cb] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:24.415 [2024/02/28 17:31:24.415 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:24.415 [2024/02/28 17:31:24.415 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_1_448039877626298924v1 is not available in any replica"] [traceID=7d4f45350e30ec9b58f1ea0401296d0b] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:24.415 [2024/02/28 17:31:24.415 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=a6b0b1b3b23298ec3cbea0dd51d897d8] [collections="[448039877626298924]"] 2024-02-29 01:31:24.417 [2024/02/28 17:31:24.417 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:24.417 [2024/02/28 17:31:24.417 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=1bc6451a32219c78e3a81fc989cdddde] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:24.420 [2024/02/28 17:31:24.420 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:24.421 [2024/02/28 17:31:24.420 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=b2953962639b26ab39577313bf0e7ab7] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:24.421 [2024/02/28 17:31:24.421 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:24.421 [2024/02/28 17:31:24.421 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=3a1d739f43b9084c0f5546e131d84a15] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:24.426 [2024/02/28 17:31:24.426 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:24.426 [2024/02/28 17:31:24.426 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=308baf2794113ca510ded0549191038c] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:24.428 [2024/02/28 17:31:24.428 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626527182] 2024-02-29 01:31:24.428 [2024/02/28 17:31:24.428 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=2eae5d6bd2243f995d2f2eeb54207082] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626527182]"] 2024-02-29 01:31:24.440 [2024/02/28 17:31:24.440 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626683] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:24.440 [2024/02/28 17:31:24.440 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626683] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:24.546 [2024/02/28 17:31:24.545 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:24.546 [2024/02/28 17:31:24.546 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=a886f5255812f0d1aabf4a6f61c13a9a] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:24.663 [2024/02/28 17:31:24.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626714] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:24.663 [2024/02/28 17:31:24.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626715] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626524214][leader=3]"] 2024-02-29 01:31:24.663 [2024/02/28 17:31:24.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626716] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:24.663 [2024/02/28 17:31:24.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626717] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626528386][leader=3]"] 2024-02-29 01:31:24.663 [2024/02/28 17:31:24.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626718] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626527785][leader=3]"] 2024-02-29 01:31:24.663 [2024/02/28 17:31:24.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626719] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:24.663 [2024/02/28 17:31:24.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626720] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:24.665 [2024/02/28 17:31:24.665 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:24.665 [2024/02/28 17:31:24.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:24.665 [2024/02/28 17:31:24.665 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:24.665 [2024/02/28 17:31:24.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:24.666 [2024/02/28 17:31:24.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:24.667 [2024/02/28 17:31:24.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626721] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759725]"] 2024-02-29 01:31:24.667 [2024/02/28 17:31:24.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626722] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760748]"] 2024-02-29 01:31:24.667 [2024/02/28 17:31:24.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626723] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761143]"] 2024-02-29 01:31:24.667 [2024/02/28 17:31:24.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626724] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760192]"] 2024-02-29 01:31:24.667 [2024/02/28 17:31:24.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626725] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760199]"] 2024-02-29 01:31:24.676 [2024/02/28 17:31:24.676 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626719] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:24.676 [2024/02/28 17:31:24.676 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626719] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:24.677 [2024/02/28 17:31:24.676 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626719] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:24.677 [2024/02/28 17:31:24.677 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626720] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:24.677 [2024/02/28 17:31:24.677 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626720] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:24.677 [2024/02/28 17:31:24.677 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626720] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:24.677 [2024/02/28 17:31:24.677 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626714] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:24.677 [2024/02/28 17:31:24.677 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626714] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:24.677 [2024/02/28 17:31:24.677 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626714] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:24.677 [2024/02/28 17:31:24.677 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626716] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:24.677 [2024/02/28 17:31:24.677 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626716] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:24.677 [2024/02/28 17:31:24.677 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626716] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:24.678 [2024/02/28 17:31:24.678 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626719] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:24.678 [2024/02/28 17:31:24.678 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626720] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:24.678 [2024/02/28 17:31:24.678 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626714] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:24.678 [2024/02/28 17:31:24.678 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626716] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:24.678 [2024/02/28 17:31:24.678 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=8] [toRemoveNum=4] 2024-02-29 01:31:24.678 [2024/02/28 17:31:24.678 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=37] [waitingTaskNum=0] [segmentTaskNum=37] [channelTaskNum=0] 2024-02-29 01:31:24.683 [2024/02/28 17:31:24.682 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626691] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:24.683 [2024/02/28 17:31:24.682 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626691] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:24.696 [2024/02/28 17:31:24.696 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626715] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:24.696 [2024/02/28 17:31:24.696 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626722] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:24.696 [2024/02/28 17:31:24.696 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626724] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:24.696 [2024/02/28 17:31:24.696 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626723] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.696 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=29] ["time cost"=361.757µs] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626725] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626683] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.696 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626717] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626683] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760430] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626718] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626721] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626702] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626702] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759770] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626693] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626691] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626693] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760366] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626689] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626689] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759628] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626686] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626686] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760744] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626685] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [meta/coordinator_broker.go:80] ["failed to get collection schema"] [error="context canceled"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626685] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759971] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [task/executor.go:538] ["failed to get collection info"] [error="context canceled"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626707] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626691] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626707] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759968] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626704] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626704] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759777] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=29] [committedNum=9] [toRemoveNum=8] 2024-02-29 01:31:24.697 [2024/02/28 17:31:24.697 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=29] [waitingTaskNum=0] [segmentTaskNum=29] [channelTaskNum=0] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626692] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626701] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626692] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626701] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626706] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626706] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626682] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626682] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626678] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626678] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626699] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626681] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626699] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626681] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626690] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626690] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626688] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626703] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626688] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626703] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626679] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626679] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626708] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626708] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626697] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626697] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626687] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626713] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626687] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:24.836 [2024/02/28 17:31:24.836 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626713] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:24.855 [2024/02/28 17:31:24.854 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626680] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:24.855 [2024/02/28 17:31:24.855 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626680] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:24.867 [2024/02/28 17:31:24.867 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626698] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:24.868 [2024/02/28 17:31:24.867 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626698] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:24.907 [2024/02/28 17:31:24.907 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626700] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:24.907 [2024/02/28 17:31:24.907 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626700] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:24.930 [2024/02/28 17:31:24.929 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:24.930 [2024/02/28 17:31:24.929 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=3d8dbb16b371f6a8c819ba7558f91894] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:24.994 [2024/02/28 17:31:24.993 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:24.994 [2024/02/28 17:31:24.993 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=550d8d38dbd981fd3de3896f04dbe6f5] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:24.994 [2024/02/28 17:31:24.994 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:24.994 [2024/02/28 17:31:24.994 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=8086cef884dd3071957ea674ac20e6e0] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:25.096 [2024/02/28 17:31:25.096 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626527182] 2024-02-29 01:31:25.096 [2024/02/28 17:31:25.096 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=68e60f780a1bf5f307e9f5609cbd34dc] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626527182]"] 2024-02-29 01:31:25.097 [2024/02/28 17:31:25.097 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=53e28694f1b0f7133ba2708b10dfcb4d] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626299027,448039877626299016,448039877626298948,448039877626298970,448039877626298981,448039877626298999,448039877626298942,448039877626298936,448039877626298953,448039877626299011,448039877626298976,448039877626299022,448039877626298931,448039877626298993,448039877626299033,448039877626298965,448039877626298959,448039877626299005,448039877626298987]"] 2024-02-29 01:31:25.097 [2024/02/28 17:31:25.097 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=53e28694f1b0f7133ba2708b10dfcb4d] [collectionID=448039877626298924] 2024-02-29 01:31:25.097 [2024/02/28 17:31:25.097 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=53e28694f1b0f7133ba2708b10dfcb4d] [collectionID=448039877626298924] 2024-02-29 01:31:25.097 [2024/02/28 17:31:25.097 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:25.098 [2024/02/28 17:31:25.098 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=53e28694f1b0f7133ba2708b10dfcb4d] [collectionID=448039877626298924] 2024-02-29 01:31:25.098 [2024/02/28 17:31:25.098 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=53e28694f1b0f7133ba2708b10dfcb4d] [collectionID=448039877626298924] 2024-02-29 01:31:25.102 [2024/02/28 17:31:25.102 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=d20f871666823e1ada0ccac001571eea] [collections="[448039877626298924]"] 2024-02-29 01:31:25.103 [2024/02/28 17:31:25.103 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:25.103 [2024/02/28 17:31:25.103 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=c0d033868713217f3d31ddad5ed57bd1] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:25.104 [2024/02/28 17:31:25.104 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:25.104 [2024/02/28 17:31:25.104 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=aac260e22e7d8d303fcd8ae5cf0ff052] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.172 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=29] [waitingTaskNum=0] [segmentTaskNum=29] [channelTaskNum=0] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.173 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=10] ["time cost"=276.84µs] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.173 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626706] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.173 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626706] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760215] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626679] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626679] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742592] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626698] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626698] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760696] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626687] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626687] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760570] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626681] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626681] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760025] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626708] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626708] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760374] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626697] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626697] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760422] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626691] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626691] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760970] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626682] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626682] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760757] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626699] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626699] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761556] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626678] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626678] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742556] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626700] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626700] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759826] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626690] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626690] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761031] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626688] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626688] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761482] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626692] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626692] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761308] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626701] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626701] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759612] 2024-02-29 01:31:25.175 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626713] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:25.176 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:25.176 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626713] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761572] 2024-02-29 01:31:25.176 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626703] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:25.176 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:25.176 [2024/02/28 17:31:25.174 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626703] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759849] 2024-02-29 01:31:25.176 [2024/02/28 17:31:25.174 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626680] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:25.176 [2024/02/28 17:31:25.174 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:25.176 [2024/02/28 17:31:25.175 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626680] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761400] 2024-02-29 01:31:25.176 [2024/02/28 17:31:25.175 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=10] [committedNum=0] [toRemoveNum=19] 2024-02-29 01:31:25.176 [2024/02/28 17:31:25.175 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=10] [waitingTaskNum=0] [segmentTaskNum=10] [channelTaskNum=0] 2024-02-29 01:31:25.201 [2024/02/28 17:31:25.201 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626721] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:25.213 [2024/02/28 17:31:25.213 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626724] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:25.232 [2024/02/28 17:31:25.231 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626725] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:25.232 [2024/02/28 17:31:25.232 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626722] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:25.232 [2024/02/28 17:31:25.232 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626723] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:25.249 [2024/02/28 17:31:25.249 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626715] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:25.282 [2024/02/28 17:31:25.274 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:25.282 [2024/02/28 17:31:25.274 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=aeefc563117dc160508a9215732b7c3c] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:25.283 [2024/02/28 17:31:25.283 +00:00] [WARN] [task/executor.go:468] ["failed to sync distribution"] [taskID=1709136626715] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:25.283 [2024/02/28 17:31:25.283 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626715] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:25.330 [2024/02/28 17:31:25.330 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626718] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:25.334 [2024/02/28 17:31:25.334 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626718] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] [elapsed=3.93343ms] 2024-02-29 01:31:25.376 [2024/02/28 17:31:25.376 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626717] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:25.382 [2024/02/28 17:31:25.381 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626717] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] [elapsed=5.771643ms] 2024-02-29 01:31:25.447 [2024/02/28 17:31:25.447 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626723] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:25.447 [2024/02/28 17:31:25.447 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626723] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:25.447 [2024/02/28 17:31:25.447 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626721] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:25.447 [2024/02/28 17:31:25.447 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626721] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:25.449 [2024/02/28 17:31:25.449 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:25.449 [2024/02/28 17:31:25.449 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=c2d32b5a443a42b26b634ad3412c8d25] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:25.468 [2024/02/28 17:31:25.467 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626722] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:25.468 [2024/02/28 17:31:25.468 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626722] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:25.505 [2024/02/28 17:31:25.504 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=20066b1104cd471a9963dc64249e83ef] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626299033,448039877626299016,448039877626298970,448039877626298936,448039877626298931,448039877626298993,448039877626299011,448039877626298942,448039877626298981,448039877626298948,448039877626298953,448039877626299027,448039877626299022,448039877626298976,448039877626298999,448039877626298965,448039877626299005,448039877626298959,448039877626298987]"] 2024-02-29 01:31:25.505 [2024/02/28 17:31:25.505 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=20066b1104cd471a9963dc64249e83ef] [collectionID=448039877626298924] 2024-02-29 01:31:25.505 [2024/02/28 17:31:25.505 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=20066b1104cd471a9963dc64249e83ef] [collectionID=448039877626298924] 2024-02-29 01:31:25.505 [2024/02/28 17:31:25.505 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:25.505 [2024/02/28 17:31:25.505 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=20066b1104cd471a9963dc64249e83ef] [collectionID=448039877626298924] 2024-02-29 01:31:25.505 [2024/02/28 17:31:25.505 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=20066b1104cd471a9963dc64249e83ef] [collectionID=448039877626298924] 2024-02-29 01:31:25.537 [2024/02/28 17:31:25.536 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626725] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:25.537 [2024/02/28 17:31:25.536 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626725] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:25.550 [2024/02/28 17:31:25.550 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=75e0d082c30b95a9b64a8f97f29ffef5] [collections="[448039877626298924]"] 2024-02-29 01:31:25.551 [2024/02/28 17:31:25.551 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:25.551 [2024/02/28 17:31:25.551 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=d40a267fffc0fce611a1a1c6973ff74d] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:25.556 [2024/02/28 17:31:25.556 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:25.556 [2024/02/28 17:31:25.556 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=64ab14bd6be02a09075fafc56c639090] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:25.568 [2024/02/28 17:31:25.568 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626724] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:25.568 [2024/02/28 17:31:25.568 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626724] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:25.588 [2024/02/28 17:31:25.586 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:25.588 [2024/02/28 17:31:25.586 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=a6ae0d56aae123fa92f88f6c2c702740] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:25.596 [2024/02/28 17:31:25.593 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:25.596 [2024/02/28 17:31:25.593 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=fec2bae93bf3ffb82746b87207a782e8] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:25.598 [2024/02/28 17:31:25.598 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:25.598 [2024/02/28 17:31:25.598 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=5675277b420d57b412505177463f8da9] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:25.605 [2024/02/28 17:31:25.605 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626523124] [error="node offline[node=8]"] 2024-02-29 01:31:25.605 [2024/02/28 17:31:25.605 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_1_448039877626298924v1 is not available in any replica"] [traceID=a10cd045e0678be73bfd1a3779c5d305] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:25.664 [2024/02/28 17:31:25.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626726] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:25.664 [2024/02/28 17:31:25.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626727] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:25.664 [2024/02/28 17:31:25.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626728] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:25.664 [2024/02/28 17:31:25.664 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626729] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:25.664 [2024/02/28 17:31:25.664 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626523124] [error="node offline[node=8]"] 2024-02-29 01:31:25.664 [2024/02/28 17:31:25.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:25.664 [2024/02/28 17:31:25.664 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:25.664 [2024/02/28 17:31:25.664 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.665 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.666 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.666 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.666 [2024/02/28 17:31:25.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.666 [2024/02/28 17:31:25.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.666 [2024/02/28 17:31:25.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.666 [2024/02/28 17:31:25.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.666 [2024/02/28 17:31:25.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.666 [2024/02/28 17:31:25.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.666 [2024/02/28 17:31:25.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.666 [2024/02/28 17:31:25.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.666 [2024/02/28 17:31:25.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.666 [2024/02/28 17:31:25.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:25.666 [2024/02/28 17:31:25.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:25.666 [2024/02/28 17:31:25.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:25.666 [2024/02/28 17:31:25.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:25.666 [2024/02/28 17:31:25.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626730] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761400]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626731] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760025]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626732] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760757]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626733] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760430]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626734] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759971]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626735] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760744]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626736] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760570]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626737] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761482]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626738] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759628]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626739] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761031]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626740] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760970]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626741] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761308]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626742] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760366]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626743] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760422]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626744] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761572]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626745] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760696]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626746] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759826]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626747] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761556]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626748] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759612]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626749] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759770]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626750] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759849]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626751] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759777]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626752] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760215]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626753] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760374]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626754] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759968]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626755] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742556]"] 2024-02-29 01:31:25.667 [2024/02/28 17:31:25.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626756] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742592]"] 2024-02-29 01:31:25.669 [2024/02/28 17:31:25.669 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626728] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:25.669 [2024/02/28 17:31:25.669 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626728] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:25.669 [2024/02/28 17:31:25.669 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626728] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:25.669 [2024/02/28 17:31:25.669 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626727] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:25.669 [2024/02/28 17:31:25.669 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626727] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:25.670 [2024/02/28 17:31:25.669 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626727] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:25.670 [2024/02/28 17:31:25.670 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626729] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:25.670 [2024/02/28 17:31:25.670 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626729] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:25.670 [2024/02/28 17:31:25.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626729] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:25.670 [2024/02/28 17:31:25.670 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626726] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:25.670 [2024/02/28 17:31:25.670 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626726] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:25.670 [2024/02/28 17:31:25.670 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626726] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:25.671 [2024/02/28 17:31:25.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626728] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:25.671 [2024/02/28 17:31:25.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626727] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:25.671 [2024/02/28 17:31:25.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626729] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:25.671 [2024/02/28 17:31:25.671 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626726] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:25.671 [2024/02/28 17:31:25.671 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=27] [toRemoveNum=4] 2024-02-29 01:31:25.671 [2024/02/28 17:31:25.671 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=37] [waitingTaskNum=0] [segmentTaskNum=37] [channelTaskNum=0] 2024-02-29 01:31:25.673 [2024/02/28 17:31:25.673 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=29] ["time cost"=929.004µs] 2024-02-29 01:31:25.673 [2024/02/28 17:31:25.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626718] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626527785] 2024-02-29 01:31:25.673 [2024/02/28 17:31:25.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626715] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626524214] 2024-02-29 01:31:25.673 [2024/02/28 17:31:25.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626717] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626528386] 2024-02-29 01:31:25.673 [2024/02/28 17:31:25.673 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626721] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:25.673 [2024/02/28 17:31:25.673 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626721] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759725] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.673 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626723] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.673 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626723] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761143] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.673 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626724] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.673 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626724] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760192] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.673 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626722] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.674 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.674 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626722] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760748] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.674 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626725] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.674 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.674 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626725] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760199] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.674 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=29] [committedNum=27] [toRemoveNum=8] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.674 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=29] [waitingTaskNum=0] [segmentTaskNum=29] [channelTaskNum=0] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626750] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626751] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.674 [2024/02/28 17:31:25.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626745] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.675 [2024/02/28 17:31:25.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626743] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.675 [2024/02/28 17:31:25.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626742] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.675 [2024/02/28 17:31:25.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626755] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.675 [2024/02/28 17:31:25.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626737] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.675 [2024/02/28 17:31:25.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626732] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.675 [2024/02/28 17:31:25.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626741] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.675 [2024/02/28 17:31:25.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626739] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.675 [2024/02/28 17:31:25.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626753] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.676 [2024/02/28 17:31:25.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626734] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.676 [2024/02/28 17:31:25.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626752] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.676 [2024/02/28 17:31:25.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626730] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.676 [2024/02/28 17:31:25.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626744] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.676 [2024/02/28 17:31:25.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626736] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.676 [2024/02/28 17:31:25.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626740] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.676 [2024/02/28 17:31:25.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626738] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.677 [2024/02/28 17:31:25.676 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626747] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.677 [2024/02/28 17:31:25.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626731] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.677 [2024/02/28 17:31:25.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626735] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.677 [2024/02/28 17:31:25.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626746] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.677 [2024/02/28 17:31:25.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626748] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.677 [2024/02/28 17:31:25.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626756] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.677 [2024/02/28 17:31:25.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626754] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.677 [2024/02/28 17:31:25.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626733] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.678 [2024/02/28 17:31:25.677 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626749] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:25.831 [2024/02/28 17:31:25.831 +00:00] [DEBUG] [querycoordv2/server.go:528] ["QueryCoord current state"] [StateCode=Healthy] 2024-02-29 01:31:26.042 [2024/02/28 17:31:26.042 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626742] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.042 [2024/02/28 17:31:26.042 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626755] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.042 [2024/02/28 17:31:26.042 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626753] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.043 [2024/02/28 17:31:26.043 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=c8cb091369cc1532986f453cec31094a] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298965,448039877626298942,448039877626298959,448039877626299027,448039877626298953,448039877626299011,448039877626298931,448039877626298981,448039877626299016,448039877626299033,448039877626299022,448039877626298993,448039877626299005,448039877626298936,448039877626298948,448039877626298999,448039877626298987,448039877626298970,448039877626298976]"] 2024-02-29 01:31:26.043 [2024/02/28 17:31:26.043 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626736] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.043 [2024/02/28 17:31:26.043 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=c8cb091369cc1532986f453cec31094a] [collectionID=448039877626298924] 2024-02-29 01:31:26.043 [2024/02/28 17:31:26.043 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=c8cb091369cc1532986f453cec31094a] [collectionID=448039877626298924] 2024-02-29 01:31:26.043 [2024/02/28 17:31:26.043 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:26.043 [2024/02/28 17:31:26.043 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626740] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.044 [2024/02/28 17:31:26.044 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626744] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.044 [2024/02/28 17:31:26.044 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626751] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.044 [2024/02/28 17:31:26.044 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=c8cb091369cc1532986f453cec31094a] [collectionID=448039877626298924] 2024-02-29 01:31:26.044 [2024/02/28 17:31:26.044 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=c8cb091369cc1532986f453cec31094a] [collectionID=448039877626298924] 2024-02-29 01:31:26.052 [2024/02/28 17:31:26.052 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=deab83d6d6e33b67cd098b231406cae9] [collections="[448039877626298924]"] 2024-02-29 01:31:26.056 [2024/02/28 17:31:26.056 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626749] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.084 [2024/02/28 17:31:26.084 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626731] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.150 [2024/02/28 17:31:26.150 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626743] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.169 [2024/02/28 17:31:26.169 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=29] [waitingTaskNum=0] [segmentTaskNum=29] [channelTaskNum=0] 2024-02-29 01:31:26.170 [2024/02/28 17:31:26.170 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=29] ["time cost"=176.928µs] 2024-02-29 01:31:26.170 [2024/02/28 17:31:26.170 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=29] [committedNum=0] [toRemoveNum=0] 2024-02-29 01:31:26.170 [2024/02/28 17:31:26.170 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=29] [waitingTaskNum=0] [segmentTaskNum=29] [channelTaskNum=0] 2024-02-29 01:31:26.183 [2024/02/28 17:31:26.182 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626741] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.196 [2024/02/28 17:31:26.192 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626735] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.196 [2024/02/28 17:31:26.194 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626733] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.209 [2024/02/28 17:31:26.209 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626748] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.209 [2024/02/28 17:31:26.209 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626745] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.209 [2024/02/28 17:31:26.209 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626734] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.292 [2024/02/28 17:31:26.291 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626739] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.292 [2024/02/28 17:31:26.291 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626738] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.292 [2024/02/28 17:31:26.291 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626754] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.292 [2024/02/28 17:31:26.291 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626752] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.292 [2024/02/28 17:31:26.291 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626750] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.292 [2024/02/28 17:31:26.292 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626747] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.447 [2024/02/28 17:31:26.447 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626732] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.448 [2024/02/28 17:31:26.447 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626737] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.548 [2024/02/28 17:31:26.548 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626736] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:26.549 [2024/02/28 17:31:26.548 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626736] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:26.663 [2024/02/28 17:31:26.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626757] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626528386][leader=3]"] 2024-02-29 01:31:26.663 [2024/02/28 17:31:26.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626758] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:26.663 [2024/02/28 17:31:26.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626759] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:26.663 [2024/02/28 17:31:26.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626760] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:26.663 [2024/02/28 17:31:26.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626761] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626524214][leader=3]"] 2024-02-29 01:31:26.663 [2024/02/28 17:31:26.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626762] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:26.663 [2024/02/28 17:31:26.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626763] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626527785][leader=3]"] 2024-02-29 01:31:26.665 [2024/02/28 17:31:26.665 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626526417] [error="node offline[node=8]"] 2024-02-29 01:31:26.665 [2024/02/28 17:31:26.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:26.665 [2024/02/28 17:31:26.665 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:26.665 [2024/02/28 17:31:26.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:26.666 [2024/02/28 17:31:26.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:26.667 [2024/02/28 17:31:26.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626764] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759725]"] 2024-02-29 01:31:26.667 [2024/02/28 17:31:26.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626765] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760748]"] 2024-02-29 01:31:26.667 [2024/02/28 17:31:26.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626766] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761143]"] 2024-02-29 01:31:26.667 [2024/02/28 17:31:26.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626767] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760192]"] 2024-02-29 01:31:26.667 [2024/02/28 17:31:26.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626768] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760199]"] 2024-02-29 01:31:26.696 [2024/02/28 17:31:26.696 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626760] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:26.696 [2024/02/28 17:31:26.696 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626760] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:26.697 [2024/02/28 17:31:26.696 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626760] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:26.697 [2024/02/28 17:31:26.697 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626759] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:26.697 [2024/02/28 17:31:26.697 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626759] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:26.697 [2024/02/28 17:31:26.697 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626759] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:26.698 [2024/02/28 17:31:26.697 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626758] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:26.698 [2024/02/28 17:31:26.698 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626758] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:26.698 [2024/02/28 17:31:26.698 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626758] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:26.698 [2024/02/28 17:31:26.698 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626762] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:26.698 [2024/02/28 17:31:26.698 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626762] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:26.698 [2024/02/28 17:31:26.698 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626762] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:26.699 [2024/02/28 17:31:26.699 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626760] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:26.699 [2024/02/28 17:31:26.699 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626759] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:26.699 [2024/02/28 17:31:26.699 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626758] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:26.699 [2024/02/28 17:31:26.699 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626762] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:26.699 [2024/02/28 17:31:26.699 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=8] [toRemoveNum=4] 2024-02-29 01:31:26.699 [2024/02/28 17:31:26.699 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=37] [waitingTaskNum=0] [segmentTaskNum=37] [channelTaskNum=0] 2024-02-29 01:31:26.699 [2024/02/28 17:31:26.699 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626746] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:26.700 [2024/02/28 17:31:26.700 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626761] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:26.700 [2024/02/28 17:31:26.700 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626763] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:26.700 [2024/02/28 17:31:26.700 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626764] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:26.700 [2024/02/28 17:31:26.700 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626767] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:26.700 [2024/02/28 17:31:26.700 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626757] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:26.700 [2024/02/28 17:31:26.700 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626766] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:26.700 [2024/02/28 17:31:26.700 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626765] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:26.700 [2024/02/28 17:31:26.700 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=36] ["time cost"=294.496µs] 2024-02-29 01:31:26.700 [2024/02/28 17:31:26.700 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626768] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:26.700 [2024/02/28 17:31:26.700 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626736] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:26.700 [2024/02/28 17:31:26.700 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:26.700 [2024/02/28 17:31:26.700 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626736] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760570] 2024-02-29 01:31:26.700 [2024/02/28 17:31:26.700 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=36] [committedNum=8] [toRemoveNum=1] 2024-02-29 01:31:26.700 [2024/02/28 17:31:26.700 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=36] [waitingTaskNum=0] [segmentTaskNum=36] [channelTaskNum=0] 2024-02-29 01:31:26.864 [2024/02/28 17:31:26.863 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626744] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:26.864 [2024/02/28 17:31:26.864 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626744] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:26.864 [2024/02/28 17:31:26.863 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626740] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:26.864 [2024/02/28 17:31:26.864 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626740] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:26.920 [2024/02/28 17:31:26.920 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626730] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:27.000 [2024/02/28 17:31:27.000 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626756] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:27.197 [2024/02/28 17:31:27.196 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=36] [waitingTaskNum=0] [segmentTaskNum=36] [channelTaskNum=0] 2024-02-29 01:31:27.198 [2024/02/28 17:31:27.198 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=34] ["time cost"=1.003509ms] 2024-02-29 01:31:27.198 [2024/02/28 17:31:27.198 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626744] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:27.198 [2024/02/28 17:31:27.198 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:27.199 [2024/02/28 17:31:27.198 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626744] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761572] 2024-02-29 01:31:27.199 [2024/02/28 17:31:27.198 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626740] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:27.199 [2024/02/28 17:31:27.198 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:27.199 [2024/02/28 17:31:27.199 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626740] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760970] 2024-02-29 01:31:27.199 [2024/02/28 17:31:27.199 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=34] [committedNum=0] [toRemoveNum=2] 2024-02-29 01:31:27.199 [2024/02/28 17:31:27.199 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=34] [waitingTaskNum=0] [segmentTaskNum=34] [channelTaskNum=0] 2024-02-29 01:31:27.296 [2024/02/28 17:31:27.295 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626732] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:27.296 [2024/02/28 17:31:27.295 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626732] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:27.296 [2024/02/28 17:31:27.295 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626742] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:27.296 [2024/02/28 17:31:27.295 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626742] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:27.296 [2024/02/28 17:31:27.295 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626755] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:27.296 [2024/02/28 17:31:27.295 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626755] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:27.296 [2024/02/28 17:31:27.295 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626751] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:27.296 [2024/02/28 17:31:27.295 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626751] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:27.296 [2024/02/28 17:31:27.295 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626753] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:27.296 [2024/02/28 17:31:27.295 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626753] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:27.296 [2024/02/28 17:31:27.295 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626731] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:27.296 [2024/02/28 17:31:27.295 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626731] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:27.296 [2024/02/28 17:31:27.296 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626743] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:27.296 [2024/02/28 17:31:27.296 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626743] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:27.339 [2024/02/28 17:31:27.339 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626749] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:27.339 [2024/02/28 17:31:27.339 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626749] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:27.339 [2024/02/28 17:31:27.339 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626735] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:27.339 [2024/02/28 17:31:27.339 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626735] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:27.339 [2024/02/28 17:31:27.339 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626747] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:27.339 [2024/02/28 17:31:27.339 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626739] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:27.339 [2024/02/28 17:31:27.339 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626747] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:27.339 [2024/02/28 17:31:27.339 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626739] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:27.339 [2024/02/28 17:31:27.339 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626752] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:27.339 [2024/02/28 17:31:27.339 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626752] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:27.339 [2024/02/28 17:31:27.339 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626754] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:27.339 [2024/02/28 17:31:27.339 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626737] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:27.339 [2024/02/28 17:31:27.339 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626745] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:27.339 [2024/02/28 17:31:27.339 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626754] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:27.339 [2024/02/28 17:31:27.339 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626737] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:27.339 [2024/02/28 17:31:27.339 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626745] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:27.404 [2024/02/28 17:31:27.404 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=4059c1fd8ba0769409093dbf871d0353] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626299005,448039877626299016,448039877626298999,448039877626299011,448039877626298936,448039877626298970,448039877626298987,448039877626298993,448039877626299027,448039877626298948,448039877626298965,448039877626299033,448039877626298981,448039877626298931,448039877626298953,448039877626299022,448039877626298959,448039877626298942,448039877626298976]"] 2024-02-29 01:31:27.404 [2024/02/28 17:31:27.404 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=4059c1fd8ba0769409093dbf871d0353] [collectionID=448039877626298924] 2024-02-29 01:31:27.404 [2024/02/28 17:31:27.404 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=4059c1fd8ba0769409093dbf871d0353] [collectionID=448039877626298924] 2024-02-29 01:31:27.404 [2024/02/28 17:31:27.404 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:27.407 [2024/02/28 17:31:27.407 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=4059c1fd8ba0769409093dbf871d0353] [collectionID=448039877626298924] 2024-02-29 01:31:27.407 [2024/02/28 17:31:27.407 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=4059c1fd8ba0769409093dbf871d0353] [collectionID=448039877626298924] 2024-02-29 01:31:27.445 [2024/02/28 17:31:27.445 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626748] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:27.445 [2024/02/28 17:31:27.445 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626741] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:27.445 [2024/02/28 17:31:27.445 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626734] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:27.445 [2024/02/28 17:31:27.445 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626748] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:27.445 [2024/02/28 17:31:27.445 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626741] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:27.445 [2024/02/28 17:31:27.445 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626734] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:27.445 [2024/02/28 17:31:27.445 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626738] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:27.445 [2024/02/28 17:31:27.445 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626738] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:27.445 [2024/02/28 17:31:27.445 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626750] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:27.445 [2024/02/28 17:31:27.445 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626750] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:27.445 [2024/02/28 17:31:27.445 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626746] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:27.445 [2024/02/28 17:31:27.445 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626746] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:27.449 [2024/02/28 17:31:27.449 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=7b5ff5da1a8aa9d5f7b9b873f21c1b7c] [collections="[448039877626298924]"] 2024-02-29 01:31:27.466 [2024/02/28 17:31:27.466 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626756] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:27.466 [2024/02/28 17:31:27.466 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626756] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:27.519 [2024/02/28 17:31:27.519 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626730] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:27.519 [2024/02/28 17:31:27.519 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626730] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:27.523 [2024/02/28 17:31:27.522 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626764] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:27.523 [2024/02/28 17:31:27.523 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626767] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:27.548 [2024/02/28 17:31:27.548 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626766] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:27.568 [2024/02/28 17:31:27.568 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626765] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:27.652 [2024/02/28 17:31:27.652 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626761] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:27.663 [2024/02/28 17:31:27.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626769] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:27.664 [2024/02/28 17:31:27.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626770] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:27.664 [2024/02/28 17:31:27.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626771] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:27.664 [2024/02/28 17:31:27.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626772] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.664 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626523124] [error="node offline[node=8]"] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626523123] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626523123]"] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.665 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.666 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.666 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.666 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.666 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.666 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.666 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.666 [2024/02/28 17:31:27.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.666 [2024/02/28 17:31:27.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:27.666 [2024/02/28 17:31:27.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:27.666 [2024/02/28 17:31:27.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:27.666 [2024/02/28 17:31:27.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:27.666 [2024/02/28 17:31:27.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:27.667 [2024/02/28 17:31:27.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626773] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760570]"] 2024-02-29 01:31:27.667 [2024/02/28 17:31:27.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626774] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760970]"] 2024-02-29 01:31:27.667 [2024/02/28 17:31:27.667 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626775] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761572]"] 2024-02-29 01:31:27.672 [2024/02/28 17:31:27.671 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626769] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:27.672 [2024/02/28 17:31:27.671 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626769] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:27.672 [2024/02/28 17:31:27.671 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626769] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:27.672 [2024/02/28 17:31:27.672 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626770] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:27.672 [2024/02/28 17:31:27.672 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626770] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:27.672 [2024/02/28 17:31:27.672 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626770] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:27.672 [2024/02/28 17:31:27.672 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626771] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:27.672 [2024/02/28 17:31:27.672 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626771] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:27.672 [2024/02/28 17:31:27.672 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626771] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:27.672 [2024/02/28 17:31:27.672 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626772] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:27.673 [2024/02/28 17:31:27.672 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626772] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:27.673 [2024/02/28 17:31:27.673 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626772] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:27.673 [2024/02/28 17:31:27.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626769] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:27.673 [2024/02/28 17:31:27.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626770] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:27.673 [2024/02/28 17:31:27.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626771] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:27.673 [2024/02/28 17:31:27.673 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626772] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:27.673 [2024/02/28 17:31:27.673 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=3] [toRemoveNum=4] 2024-02-29 01:31:27.673 [2024/02/28 17:31:27.673 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=37] [waitingTaskNum=0] [segmentTaskNum=37] [channelTaskNum=0] 2024-02-29 01:31:27.675 [2024/02/28 17:31:27.674 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626774] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:27.675 [2024/02/28 17:31:27.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626775] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:27.675 [2024/02/28 17:31:27.675 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626773] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:27.675 [2024/02/28 17:31:27.675 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=14] ["time cost"=639.549µs] 2024-02-29 01:31:27.675 [2024/02/28 17:31:27.675 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626748] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759612] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:27.675 [2024/02/28 17:31:27.675 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759612/448039877626577334]"] 2024-02-29 01:31:27.675 [2024/02/28 17:31:27.675 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626748] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759612] 2024-02-29 01:31:27.675 [2024/02/28 17:31:27.675 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626756] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742592] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:27.676 [2024/02/28 17:31:27.675 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742592/448039877626558818]"] 2024-02-29 01:31:27.676 [2024/02/28 17:31:27.675 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626756] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742592] 2024-02-29 01:31:27.676 [2024/02/28 17:31:27.675 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626754] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759968] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:27.676 [2024/02/28 17:31:27.675 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759968/448039877626577334]"] 2024-02-29 01:31:27.676 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626754] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759968] 2024-02-29 01:31:27.676 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626749] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759770] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:27.676 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759770/448039877626577334]"] 2024-02-29 01:31:27.676 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626749] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759770] 2024-02-29 01:31:27.676 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626750] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759849] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:27.676 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759849/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626750] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759849] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626751] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759777] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759777/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626751] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759777] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626745] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760696] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760696/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626745] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760696] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626743] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760422] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760422/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626743] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760422] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626742] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760366] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760366/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626742] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760366] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626755] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632742556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632742556/448039877626558600]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626755] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632742556] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626737] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761482] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761482/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626737] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761482] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626732] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760757] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760757/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626732] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760757] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626741] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761308] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761308/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626741] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761308] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626739] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761031] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761031/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626739] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761031] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626753] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760374] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760374/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626753] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760374] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626734] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759971] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759971/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626734] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759971] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626752] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760215] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760215/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626752] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760215] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626730] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761400] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761400/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626730] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761400] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626738] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759628] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759628/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626738] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759628] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626747] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761556] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761556/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626747] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761556] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626731] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760025] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760025/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626731] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760025] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.676 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626735] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760744] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.677 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760744/448039877626577334]"] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.677 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626735] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760744] 2024-02-29 01:31:27.677 [2024/02/28 17:31:27.677 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626746] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759826] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:27.678 [2024/02/28 17:31:27.677 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759826/448039877626577334]"] 2024-02-29 01:31:27.678 [2024/02/28 17:31:27.677 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626746] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759826] 2024-02-29 01:31:27.678 [2024/02/28 17:31:27.677 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=14] [committedNum=3] [toRemoveNum=23] 2024-02-29 01:31:27.678 [2024/02/28 17:31:27.677 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=14] [waitingTaskNum=0] [segmentTaskNum=14] [channelTaskNum=0] 2024-02-29 01:31:27.698 [2024/02/28 17:31:27.698 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626768] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:27.732 [2024/02/28 17:31:27.732 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626733] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:27.732 [2024/02/28 17:31:27.732 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626767] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:27.732 [2024/02/28 17:31:27.732 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626733] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:27.732 [2024/02/28 17:31:27.732 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626767] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:27.786 [2024/02/28 17:31:27.786 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626764] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:27.786 [2024/02/28 17:31:27.786 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626764] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:27.786 [2024/02/28 17:31:27.786 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626765] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:27.786 [2024/02/28 17:31:27.786 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626765] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:27.797 [2024/02/28 17:31:27.797 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626766] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:27.797 [2024/02/28 17:31:27.797 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626766] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:27.853 [2024/02/28 17:31:27.853 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626768] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:27.853 [2024/02/28 17:31:27.853 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626768] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:27.906 [2024/02/28 17:31:27.906 +00:00] [WARN] [task/executor.go:468] ["failed to sync distribution"] [taskID=1709136626761] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626524214] [leader=3] [node=3] [source=leader_checker] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:27.906 [2024/02/28 17:31:27.906 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626761] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877626524214/448039877626554003]"] 2024-02-29 01:31:27.937 [2024/02/28 17:31:27.936 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626757] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:27.940 [2024/02/28 17:31:27.940 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626757] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626528386] [leader=3] [node=3] [source=leader_checker] [elapsed=3.394985ms] 2024-02-29 01:31:27.949 [2024/02/28 17:31:27.949 +00:00] [INFO] [task/executor.go:464] ["Sync Distribution..."] [taskID=1709136626763] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] 2024-02-29 01:31:27.953 [2024/02/28 17:31:27.953 +00:00] [INFO] [task/executor.go:473] ["sync distribution done"] [taskID=1709136626763] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877626527785] [leader=3] [node=3] [source=leader_checker] [elapsed=4.274685ms] 2024-02-29 01:31:28.130 [2024/02/28 17:31:28.129 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=d3163033cbfdd0d1bfe21420d3684c2c] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626298936,448039877626298981,448039877626299033,448039877626298953,448039877626298970,448039877626298999,448039877626298948,448039877626299027,448039877626299022,448039877626298993,448039877626298942,448039877626298987,448039877626299005,448039877626298931,448039877626299011,448039877626298965,448039877626298959,448039877626298976,448039877626299016]"] 2024-02-29 01:31:28.130 [2024/02/28 17:31:28.130 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=d3163033cbfdd0d1bfe21420d3684c2c] [collectionID=448039877626298924] 2024-02-29 01:31:28.130 [2024/02/28 17:31:28.130 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=d3163033cbfdd0d1bfe21420d3684c2c] [collectionID=448039877626298924] 2024-02-29 01:31:28.130 [2024/02/28 17:31:28.130 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:28.130 [2024/02/28 17:31:28.130 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=d3163033cbfdd0d1bfe21420d3684c2c] [collectionID=448039877626298924] 2024-02-29 01:31:28.130 [2024/02/28 17:31:28.130 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=d3163033cbfdd0d1bfe21420d3684c2c] [collectionID=448039877626298924] 2024-02-29 01:31:28.136 [2024/02/28 17:31:28.136 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=b8d5faa0c5ad6344c8d29297d1afd5d3] [collections="[448039877626298924]"] 2024-02-29 01:31:28.176 [2024/02/28 17:31:28.175 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=14] [waitingTaskNum=0] [segmentTaskNum=14] [channelTaskNum=0] 2024-02-29 01:31:28.183 [2024/02/28 17:31:28.182 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=5] ["time cost"=196.649µs] 2024-02-29 01:31:28.183 [2024/02/28 17:31:28.182 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626757] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626528386] 2024-02-29 01:31:28.183 [2024/02/28 17:31:28.183 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626761] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877626524214] 2024-02-29 01:31:28.183 [2024/02/28 17:31:28.183 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626763] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=succeeded] [segmentID=448039877626527785] 2024-02-29 01:31:28.183 [2024/02/28 17:31:28.183 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626733] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760430] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:28.183 [2024/02/28 17:31:28.183 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760430/448039877626577334]"] 2024-02-29 01:31:28.183 [2024/02/28 17:31:28.183 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626733] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760430] 2024-02-29 01:31:28.183 [2024/02/28 17:31:28.183 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626767] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760192] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:28.183 [2024/02/28 17:31:28.183 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760192/448039877626577334]"] 2024-02-29 01:31:28.183 [2024/02/28 17:31:28.183 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626767] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760192] 2024-02-29 01:31:28.183 [2024/02/28 17:31:28.183 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626764] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632759725] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:28.183 [2024/02/28 17:31:28.183 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632759725/448039877626577334]"] 2024-02-29 01:31:28.183 [2024/02/28 17:31:28.183 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626764] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632759725] 2024-02-29 01:31:28.183 [2024/02/28 17:31:28.183 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626766] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761143] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:28.184 [2024/02/28 17:31:28.183 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761143/448039877626577334]"] 2024-02-29 01:31:28.184 [2024/02/28 17:31:28.183 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626766] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761143] 2024-02-29 01:31:28.184 [2024/02/28 17:31:28.183 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626768] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760199] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:28.184 [2024/02/28 17:31:28.183 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760199/448039877626577334]"] 2024-02-29 01:31:28.184 [2024/02/28 17:31:28.183 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626768] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760199] 2024-02-29 01:31:28.184 [2024/02/28 17:31:28.183 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626765] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760748] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:28.184 [2024/02/28 17:31:28.183 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760748/448039877626577334]"] 2024-02-29 01:31:28.184 [2024/02/28 17:31:28.183 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626765] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760748] 2024-02-29 01:31:28.184 [2024/02/28 17:31:28.183 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=5] [committedNum=0] [toRemoveNum=9] 2024-02-29 01:31:28.184 [2024/02/28 17:31:28.183 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=5] [waitingTaskNum=0] [segmentTaskNum=5] [channelTaskNum=0] 2024-02-29 01:31:28.311 [2024/02/28 17:31:28.311 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626774] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:28.326 [2024/02/28 17:31:28.326 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626773] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:28.326 [2024/02/28 17:31:28.326 +00:00] [INFO] [task/executor.go:202] ["load segments..."] [taskID=1709136626775] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] 2024-02-29 01:31:28.489 [2024/02/28 17:31:28.488 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626774] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:28.489 [2024/02/28 17:31:28.488 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626774] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:28.519 [2024/02/28 17:31:28.519 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626775] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:28.519 [2024/02/28 17:31:28.519 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626775] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:28.563 [2024/02/28 17:31:28.562 +00:00] [WARN] [task/executor.go:206] ["failed to load segment"] [taskID=1709136626773] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [node=3] [source=segment_checker] [shardLeader=3] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:28.563 [2024/02/28 17:31:28.562 +00:00] [INFO] [task/executor.go:128] ["execute action done, remove it"] [taskID=1709136626773] [step=0] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:28.620 [2024/02/28 17:31:28.620 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:28.620 [2024/02/28 17:31:28.620 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=c5a78efdb201b726e2f4d8fc04261ae0] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:28.625 [2024/02/28 17:31:28.625 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626526093] 2024-02-29 01:31:28.625 [2024/02/28 17:31:28.625 +00:00] [WARN] [querycoordv2/services.go:904] ["channel by-dev-rootcoord-dml_0_448039877626298924v0 is not available in any replica"] [traceID=989d6cea84b79a0e3fd30786204ea72b] [collectionID=448039877626298924] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626526093]"] 2024-02-29 01:31:28.654 [2024/02/28 17:31:28.654 +00:00] [INFO] [querycoordv2/services.go:198] ["load collection request received"] [traceID=091672eedecc7c4733230dc20830accc] [collectionID=448039877626298924] [replicaNumber=1] [resourceGroups="[]"] [refreshMode=false] [schema="name:\"fouram_YZpI6xUx\" fields: fields: > fields: > fields: > fields: > fields: fields: fields: fields: fields: fields: fields: > fields: fields: fields: fields: fields: fields: fields: fields: > fields: "] [fieldIndexes="[448039877626299022,448039877626298970,448039877626298959,448039877626298976,448039877626299011,448039877626298981,448039877626299016,448039877626299027,448039877626298931,448039877626298936,448039877626298993,448039877626299033,448039877626299005,448039877626298942,448039877626298953,448039877626298948,448039877626298965,448039877626298999,448039877626298987]"] 2024-02-29 01:31:28.654 [2024/02/28 17:31:28.654 +00:00] [INFO] [job/scheduler.go:150] ["start to pre-execute job"] [traceID=091672eedecc7c4733230dc20830accc] [collectionID=448039877626298924] 2024-02-29 01:31:28.654 [2024/02/28 17:31:28.654 +00:00] [INFO] [job/scheduler.go:158] ["start to execute job"] [traceID=091672eedecc7c4733230dc20830accc] [collectionID=448039877626298924] 2024-02-29 01:31:28.654 [2024/02/28 17:31:28.654 +00:00] [INFO] [meta/failed_load_cache.go:107] ["FailedLoadCache removes cache"] [collectionID=448039877626298924] 2024-02-29 01:31:28.655 [2024/02/28 17:31:28.654 +00:00] [INFO] [job/scheduler.go:144] ["start to post-execute job"] [traceID=091672eedecc7c4733230dc20830accc] [collectionID=448039877626298924] 2024-02-29 01:31:28.655 [2024/02/28 17:31:28.655 +00:00] [INFO] [job/scheduler.go:146] ["job finished"] [traceID=091672eedecc7c4733230dc20830accc] [collectionID=448039877626298924] 2024-02-29 01:31:28.660 [2024/02/28 17:31:28.660 +00:00] [INFO] [querycoordv2/services.go:58] ["show collections request received"] [traceID=d0e84a2dd22417f9b1143d759e4eb26f] [collections="[448039877626298924]"] 2024-02-29 01:31:28.663 [2024/02/28 17:31:28.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626776] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626524214][leader=3]"] 2024-02-29 01:31:28.663 [2024/02/28 17:31:28.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626777] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626551727][leader=3]"] 2024-02-29 01:31:28.663 [2024/02/28 17:31:28.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626778] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626543860][leader=3]"] 2024-02-29 01:31:28.663 [2024/02/28 17:31:28.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626779] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626527785][leader=3]"] 2024-02-29 01:31:28.663 [2024/02/28 17:31:28.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626780] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626554390][leader=3]"] 2024-02-29 01:31:28.663 [2024/02/28 17:31:28.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626781] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626555509][leader=3]"] 2024-02-29 01:31:28.663 [2024/02/28 17:31:28.663 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626782] [type=Grow] [source=leader_checker] [reason=add segment to leader view] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=High] [actionsCount=1] [actions={[type=Grow][node=3]}] [segmentID=448039877626528386][leader=3]"] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [checkers/util.go:58] ["leader is not available due to QueryNode unavailable"] [leaderID=3] [segmentID=448039877626523124] [error="node offline[node=8]"] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [error="node offline[node=8]"] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [checkers/util.go:73] ["leader is not available due to lack of segment"] [leaderID=3] [segmentID=448039877626524214] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [checkers/channel_checker.go:195] ["replica has unavailable shard leader"] [collectionID=448039877626298924] [replicaID=448039883020173313] [leaderID=3] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [error="segment lacks[segment=448039877626524214]"] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761400] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760025] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760757] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760430] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759725] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759971] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760744] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760570] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761482] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759628] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761031] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760970] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761308] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760748] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760366] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.665 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760422] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761143] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761572] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.665 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760192] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760696] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759826] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632761556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759612] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759770] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759849] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759777] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760199] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760215] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632759968] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632760374] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_0_448039877626298924v0] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626526417] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877626548132] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742556] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [balance/utils.go:70] ["create segment task"] [collection=448039877626298924] [segmentID=448039877632742592] [replica=448039883020173313] [channel=by-dev-rootcoord-dml_1_448039877626298924v1] [from=-1] [to=3] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626783] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761400]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626784] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760025]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626785] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760757]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626786] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760430]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626787] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759725]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626788] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759971]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626789] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760744]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626790] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761482]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626791] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759628]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626792] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761031]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626793] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761308]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626794] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760748]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626795] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760366]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626796] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760422]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626797] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761143]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626798] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760192]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626799] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760696]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626800] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759826]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626801] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632761556]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626802] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759612]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626803] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759770]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626804] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759849]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626805] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759777]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626806] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760199]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626807] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760215]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626808] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632759968]"] 2024-02-29 01:31:28.666 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626809] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632760374]"] 2024-02-29 01:31:28.667 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626810] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742556]"] 2024-02-29 01:31:28.667 [2024/02/28 17:31:28.666 +00:00] [INFO] [task/scheduler.go:281] ["task added"] [task="[id=1709136626811] [type=Grow] [source=segment_checker] [reason=lacks of segment] [collectionID=448039877626298924] [replicaID=448039883020173313] [priority=Normal] [actionsCount=1] [actions={[type=Grow][node=3][streaming=false]}] [segmentID=448039877632742592]"] 2024-02-29 01:31:28.680 [2024/02/28 17:31:28.680 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626781] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626555509] [taskType=Grow] 2024-02-29 01:31:28.681 [2024/02/28 17:31:28.680 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626781] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:28.681 [2024/02/28 17:31:28.680 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626781] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626555509]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626555509]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:28.681 [2024/02/28 17:31:28.681 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626778] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626543860] [taskType=Grow] 2024-02-29 01:31:28.681 [2024/02/28 17:31:28.681 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626778] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:28.681 [2024/02/28 17:31:28.681 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626778] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626543860]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626543860]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:28.681 [2024/02/28 17:31:28.681 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626780] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626554390] [taskType=Grow] 2024-02-29 01:31:28.681 [2024/02/28 17:31:28.681 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626780] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:28.681 [2024/02/28 17:31:28.681 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626780] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626554390]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626554390]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:28.681 [2024/02/28 17:31:28.681 +00:00] [WARN] [task/scheduler.go:923] ["task stale due to the segment to load not exists in targets"] [taskID=1709136626777] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [leaderID=3] [segment=448039877626551727] [taskType=Grow] 2024-02-29 01:31:28.681 [2024/02/28 17:31:28.681 +00:00] [INFO] [task/scheduler.go:450] ["failed to promote task"] [taskID=1709136626777] [collectionID=448039877626298924] [replicaID=448039883020173313] [source=leader_checker] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:28.681 [2024/02/28 17:31:28.681 +00:00] [WARN] [task/scheduler.go:416] ["failed to promote task"] [taskID=1709136626777] [error="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]"] [errorVerbose="target doesn't contain this segment: segment reduplicates[segment=448039877626551727]\n(1) attached stack trace\n -- stack trace:\n | github.com/milvus-io/milvus/pkg/util/merr.WrapErrSegmentReduplicate\n | \t/go/src/github.com/milvus-io/milvus/pkg/util/merr/utils.go:652\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkLeaderTaskStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:927\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).checkStale\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:818\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).check\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:726\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).promote\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:449\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll.func1\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:412\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskQueue).Range\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:129\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).tryPromoteAll\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:411\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).schedule\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:551\n | github.com/milvus-io/milvus/internal/querycoordv2/task.(*taskScheduler).Dispatch\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/task/scheduler.go:467\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).handleDistResp\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:105\n | github.com/milvus-io/milvus/internal/querycoordv2/dist.(*distHandler).start\n | \t/go/src/github.com/milvus-io/milvus/internal/querycoordv2/dist/dist_handler.go:81\n | runtime.goexit\n | \t/usr/local/go/src/runtime/asm_amd64.s:1598\nWraps: (2) target doesn't contain this segment\nWraps: (3) segment reduplicates[segment=448039877626551727]\nError types: (1) *withstack.withStack (2) *errutil.withPrefix (3) merr.milvusError"] 2024-02-29 01:31:28.684 [2024/02/28 17:31:28.683 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626781] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626555509] 2024-02-29 01:31:28.684 [2024/02/28 17:31:28.683 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626778] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626543860] 2024-02-29 01:31:28.684 [2024/02/28 17:31:28.683 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626780] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626554390] 2024-02-29 01:31:28.684 [2024/02/28 17:31:28.683 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626777] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=canceled] [segmentID=448039877626551727] 2024-02-29 01:31:28.684 [2024/02/28 17:31:28.683 +00:00] [DEBUG] [task/scheduler.go:435] ["promoted tasks"] [promotedNum=32] [toRemoveNum=4] 2024-02-29 01:31:28.684 [2024/02/28 17:31:28.683 +00:00] [DEBUG] [task/scheduler.go:553] ["process tasks related to node"] [nodeID=3] [processingTaskNum=37] [waitingTaskNum=0] [segmentTaskNum=37] [channelTaskNum=0] 2024-02-29 01:31:28.687 [2024/02/28 17:31:28.687 +00:00] [DEBUG] [funcutil/parallel.go:54] [process] [total=34] ["time cost"=982.94µs] 2024-02-29 01:31:28.687 [2024/02/28 17:31:28.687 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626775] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632761572] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:28.687 [2024/02/28 17:31:28.687 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632761572/448039877626577334]"] 2024-02-29 01:31:28.687 [2024/02/28 17:31:28.687 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626775] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632761572] 2024-02-29 01:31:28.687 [2024/02/28 17:31:28.687 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626773] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760570] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:28.687 [2024/02/28 17:31:28.687 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760570/448039877626577334]"] 2024-02-29 01:31:28.687 [2024/02/28 17:31:28.687 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626773] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760570] 2024-02-29 01:31:28.687 [2024/02/28 17:31:28.687 +00:00] [WARN] [task/scheduler.go:749] ["task scheduler recordSegmentTaskError"] [taskID=1709136626774] [collectionID=448039877626298924] [replicaID=448039883020173313] [segmentID=448039877632760970] [status=failed] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:28.687 [2024/02/28 17:31:28.687 +00:00] [WARN] [meta/failed_load_cache.go:97] ["FailedLoadCache put failed record"] [collectionID=448039877626298924] [error="The specified key does not exist.: key not found[key=file/delta_log/448039877626298924/448039877626298925/448039877632760970/448039877626577334]"] 2024-02-29 01:31:28.687 [2024/02/28 17:31:28.687 +00:00] [INFO] [task/scheduler.go:795] ["task removed"] [taskID=1709136626774] [collectionID=448039877626298924] [replicaID=448039883020173313] [status=failed] [segmentID=448039877632760970] 2024-02-29 01:31:28.687 [2024/02/28 17:31:28.687 +00:00] [INFO] [task/scheduler.go:588] ["processed tasks"] [nodeID=3] [toProcessNum=34] [committedNum=32] [toRemoveNum=3] 2024-02-29 01:31:28.687 [2024/02/28 17:31:28.687 +00:00] [INFO] [task/scheduler.go:594] ["process tasks related to node done"] [nodeID=3] [processingTaskNum=34] [waitingTaskNum=0] [segmentTaskNum=34] [channelTaskNum=0] 2024-02-29 01:31:28.687 [2024/02/28 17:31:28.687 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626806] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.688 [2024/02/28 17:31:28.688 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626779] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:28.689 [2024/02/28 17:31:28.688 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626776] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:28.689 [2024/02/28 17:31:28.688 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626782] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=leader_checker] 2024-02-29 01:31:28.689 [2024/02/28 17:31:28.688 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626796] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.689 [2024/02/28 17:31:28.688 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626798] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.689 [2024/02/28 17:31:28.689 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626801] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.689 [2024/02/28 17:31:28.689 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626797] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.689 [2024/02/28 17:31:28.689 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626808] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.689 [2024/02/28 17:31:28.689 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626795] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.689 [2024/02/28 17:31:28.689 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626786] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.690 [2024/02/28 17:31:28.689 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626784] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626802] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626803] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626799] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626785] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626793] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626790] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626787] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626805] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626791] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626794] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626789] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626807] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626783] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626800] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626809] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626788] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626804] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626810] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626792] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker] 2024-02-29 01:31:28.695 [2024/02/28 17:31:28.690 +00:00] [INFO] [task/executor.go:110] ["execute the action of task"] [taskID=1709136626811] [collectionID=448039877626298924] [replicaID=448039883020173313] [step=0] [source=segment_checker]