etcd cluster is not healthy

[edwinkullu]

I have tried to start the etcd but was not able to run due to this i was not able to create k8 cluster below are the error an any one help this?

Feb 19 10:09:24 master1 etcd[32723]: {"level":"warn","ts":"2025-02-19T10:09:24.597638Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"802.670971ms","expected-duration":"100ms","prefix":"","request":"header:<ID:4741609936946472385 username:"etcd-node-master1" auth_revision:1 > txn:<compare:<target:MOD key:"/registry/masterleases/172.16.16.93" mod_revision:0 > success:<request_put:<key:"/registry/masterleases/172.16.16.93" value_size:65 lease:2023687562128247129 >> failure:<>>","response":"size:16"}
Feb 19 10:09:24 master1 etcd[32723]: {"level":"info","ts":"2025-02-19T10:09:24.597767Z","caller":"traceutil/trace.go:171","msg":"trace[1737555026] transaction","detail":"{read_only:false; response_revision:6647; number_of_response:1; }","duration":"3.068326511s","start":"2025-02-19T10:09:21.529430Z","end":"2025-02-19T10:09:24.597756Z","steps":["trace[1737555026] 'process raft request' (duration: 3.068268334s)"],"step_count":1}
Feb 19 10:09:24 master1 etcd[32723]: {"level":"warn","ts":"2025-02-19T10:09:24.597826Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2025-02-19T10:09:21.529414Z","time spent":"3.068378315s","remote":"172.16.16.86:56150","response type":"/etcdserverpb.KV/Txn","request count":1,"request size":672,"response count":0,"response size":38,"request content":"compare:<target:MOD key:"/registry/leases/kube-system/apiserver-bskcrn2i4gf5c5gco6huepssle" mod_revision:6635 > success:<request_put:<key:"/registry/leases/kube-system/apiserver-bskcrn2i4gf5c5gco6huepssle" value_size:599 >> failure:<request_range:<key:"/registry/leases/kube-system/apiserver-bskcrn2i4gf5c5gco6huepssle" > >"}
Feb 19 10:09:24 master1 etcd[32723]: {"level":"info","ts":"2025-02-19T10:09:24.597956Z","caller":"traceutil/trace.go:171","msg":"trace[306317173] linearizableReadLoop","detail":"{readStateIndex:7980; appliedIndex:7979; }","duration":"3.602137215s","start":"2025-02-19T10:09:20.995810Z","end":"2025-02-19T10:09:24.597947Z","steps":["trace[306317173] 'read index received' (duration: 2.798910045s)","trace[306317173] 'applied index is now lower than readState.Index' (duration: 803.226259ms)"],"step_count":2}
Feb 19 10:09:24 master1 etcd[32723]: {"level":"warn","ts":"2025-02-19T10:09:24.597996Z","caller":"etcdserver/v3_server.go:897","msg":"ignored out-of-date read index response; local node read indexes queueing up and waiting to be in sync with leader","sent-request-id":2023687562128247136,"received-request-id":2023687562128247134}
Feb 19 10:09:24 master1 etcd[32723]: {"level":"warn","ts":"2025-02-19T10:09:24.598297Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"6.0908558s","expected-duration":"100ms","prefix":"read-only range ","request":"key:"/registry/masterleases/172.16.16.86" limit:1 ","response":"range_response_count:0 size:5"}
Feb 19 10:09:24 master1 etcd[32723]: {"level":"info","ts":"2025-02-19T10:09:24.598330Z","caller":"traceutil/trace.go:171","msg":"trace[2125538508] range","detail":"{range_begin:/registry/masterleases/172.16.16.86; range_end:; response_count:0; response_revision:6647; }","duration":"6.090911824s","start":"2025-02-19T10:09:18.507409Z","end":"2025-02-19T10:09:24.598321Z","steps":["trace[2125538508] 'agreement among raft nodes before linearized reading' (duration: 6.090855365s)"],"step_count":1}
Feb 19 10:09:24 master1 etcd[32723]: {"level":"warn","ts":"2025-02-19T10:09:24.598352Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2025-02-19T10:09:18.507394Z","time spent":"6.090952965s","remote":"172.16.16.86:55950","response type":"/etcdserverpb.KV/Range","request count":0,"request size":39,"response count":0,"response size":27,"request content":"key:"/registry/masterleases/172.16.16.86" limit:1 "}
Feb 19 10:09:24 master1 etcd[32723]: {"level":"warn","ts":"2025-02-19T10:09:24.598509Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"5.970796248s","expected-duration":"100ms","prefix":"read-only range ","request":"key:"/registry/csinodes/" range_end:"/registry/csinodes0" count_only:true ","response":"range_response_count:0 size:7"}
Feb 19 10:09:24 master1 etcd[32723]: {"level":"info","ts":"2025-02-19T10:09:24.598530Z","caller":"traceutil/trace.go:171","msg":"trace[613760895] range","detail":"{range_begin:/registry/csinodes/; range_end:/registry/csinodes0; response_count:0; response_revision:6647; }","duration":"5.970846945s","start":"2025-02-19T10:09:18.627678Z","end":"2025-02-19T10:09:24.598525Z","steps":["trace[613760895] 'agreement among raft nodes before linearized reading' (duration: 5.970796519s)"],"step_count":1}
Feb 19 10:09:24 master1 etcd[32723]: {"level":"warn","ts":"2025-02-19T10:09:24.598550Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2025-02-19T10:09:18.627663Z","time spent":"5.970879577s","remote":"172.16.16.86:56260","response type":"/etcdserverpb.KV/Range","request count":0,"request size":44,"response count":2,"response size":29,"request content":"key:"/registry/csinodes/" range_end:"/registry/csinodes0" count_only:true "}
Feb 19 10:09:24 master1 etcd[32723]: {"level":"warn","ts":"2025-02-19T10:09:24.598901Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"4.12146812s","expected-duration":"100ms","prefix":"read-only range ","request":"key:"/registry/events/kube-system/kube-scheduler-master2.182592f7eb7ab0e8" limit:1 ","response":"range_response_count:1 size:832"}
Feb 19 10:09:24 master1 etcd[32723]: {"level":"info","ts":"2025-02-19T10:09:24.598923Z","caller":"traceutil/trace.go:171","msg":"trace[1472536057] range","detail":"{range_begin:/registry/events/kube-system/kube-scheduler-master2.182592f7eb7ab0e8; range_end:; response_count:1; response_revision:6647; }","duration":"4.12153245s","start":"2025-02-19T10:09:20.477384Z","end":"2025-02-19T10:09:24.598917Z","steps":["trace[1472536057] 'agreement among raft nodes before linearized reading' (duration: 4.120661131s)"],"step_count":1}
Feb 19 10:09:24 master1 etcd[32723]: {"level":"warn","ts":"2025-02-19T10:09:24.598940Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2025-02-19T10:09:20.477357Z","time spent":"4.121578404s","remote":"172.16.16.86:55996","response type":"/etcdserverpb.KV/Range","request count":0,"request size":72,"response count":1,"response size":854,"request content":"key:"/registry/events/kube-system/kube-scheduler-master2.182592f7eb7ab0e8" limit:1 "}
Feb 19 10:09:25 master1 etcd[32723]: {"level":"warn","ts":"2025-02-19T10:09:25.098263Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2023687562128247136,"retry-timeout":"500ms"}
Feb 19 10:09:25 master1 etcd[32723]: {"level":"warn","ts":"2025-02-19T10:09:25.598355Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2023687562128247136,"retry-timeout":"500ms"}
Feb 19 10:09:25 master1 etcd[32723]: {"level":"warn","ts":"2025-02-19T10:09:25.914403Z","caller":"wal/wal.go:805","msg":"slow fdatasync","took":"1.004541701s","expected-duration":"1s"}
Feb 19 10:09:26 master1 etcd[32723]: {"level":"warn","ts":"2025-02-19T10:09:26.099185Z","caller":"etcdserver/v3_server.go:920","msg":"waiting for ReadIndex response took too long, retrying","sent-request-id":2023687562128247136,"retry-timeout":"500ms"}
Feb 19 10:09:26 master1 etcd[32723]: {"level":"warn","ts":"2025-02-19T10:09:26.124023Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2025-02-19T10:09:24.966983Z","time spent":"1.157035267s","remote":"172.16.16.59:38374","response type":"/etcdserverpb.Maintenance/Status","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""}
Feb 19 10:09:26 master1 etcd[32723]: {"level":"warn","ts":"2025-02-19T10:09:26.124240Z","caller":"v3rpc/interceptor.go:197","msg":"request stats","start time":"2025-02-19T10:09:24.970259Z","time spent":"1.153978962s","remote":"172.16.16.59:38380","response type":"/etcdserverpb.Maintenance/Status","request count":-1,"request size":-1,"response count":-1,"response size":-1,"request content":""}
Feb 19 10:09:26 master1 etcd[32723]: {"level":"warn","ts":"2025-02-19T10:09:26.124629Z","caller":"etcdserver/util.go:170","msg":"apply request took too long","took":"209.249749ms","expected-duration":"100ms","prefix":"","request":"header:<ID:11270140546775113420 username:"etcd-node-master1" auth_revision:1 > txn:<compare:<target:MOD key:"/registry/events/kube-system/kube-scheduler-master1.182591ebe0f6543a" mod_revision:6294 > success:<request_put:<key:"/registry/events/kube-system/kube-scheduler-master1.182591ebe0f6543a" value_size:635 lease:2023687562128247079 >> failure:<>>","response":"size:16"}