ConcurrentModificationException: mutation occurred during iteration

[Chillax-0v0]

server.log:

[2023-11-22 07:50:37,049] ERROR failed to complete delayed actions (kafka.server.ActionQueue)
java.util.ConcurrentModificationException: mutation occurred during iteration
	at scala.collection.mutable.MutationTracker$.checkMutations(MutationTracker.scala:43)
	at scala.collection.mutable.CheckedIndexedSeqView$CheckedIterator.hasNext(CheckedIndexedSeqView.scala:47)
	at scala.collection.StrictOptimizedIterableOps.map(StrictOptimizedIterableOps.scala:99)
	at scala.collection.StrictOptimizedIterableOps.map$(StrictOptimizedIterableOps.scala:87)
	at scala.collection.mutable.ArrayBuffer.map(ArrayBuffer.scala:43)
	at kafka.server.DelayedFetch.onComplete(DelayedFetch.scala:176)
	at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:72)
	at kafka.server.DelayedFetch.tryComplete(DelayedFetch.scala:145)
	at kafka.server.DelayedOperation.safeTryComplete(DelayedOperation.scala:121)
	at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:371)
	at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:264)
	at kafka.server.ReplicaManager.$anonfun$appendRecords$6(ReplicaManager.scala:676)
	at scala.collection.mutable.HashMap$Node.foreach(HashMap.scala:627)
	at scala.collection.mutable.HashMap.foreach(HashMap.scala:489)
	at kafka.server.ReplicaManager.$anonfun$appendRecords$5(ReplicaManager.scala:665)
	at kafka.server.ActionQueue.tryCompleteActions(ActionQueue.scala:49)
	at kafka.server.ReplicaManager.tryCompleteActions(ReplicaManager.scala:623)
	at kafka.server.KafkaApis.handle(KafkaApis.scala:328)
	at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:77)
	at java.base/java.lang.Thread.run(Thread.java:833)
[2023-11-22 07:50:38,092] WARN [BACKOFF] log cache size 1073777760 is larger than 1073741824 (com.automq.stream.s3.S3Storage)
[2023-11-22 07:50:38,132] WARN try drain backoff record fail, still backoff (com.automq.stream.s3.S3Storage)
[2023-11-22 07:50:38,232] WARN try drain backoff record fail, still backoff (com.automq.stream.s3.S3Storage)
[2023-11-22 07:50:38,332] WARN try drain backoff record fail, still backoff (com.automq.stream.s3.S3Storage)
[2023-11-22 07:50:38,432] WARN try drain backoff record fail, still backoff (com.automq.stream.s3.S3Storage)
[2023-11-22 07:50:38,532] WARN try drain backoff record fail, still backoff (com.automq.stream.s3.S3Storage)
[2023-11-22 07:50:40,938] ERROR delay fetch fail (kafka.cluster.DelayedOperations)
java.util.ConcurrentModificationException: mutation occurred during iteration
	at scala.collection.mutable.MutationTracker$.checkMutations(MutationTracker.scala:43)
	at scala.collection.mutable.CheckedIndexedSeqView$CheckedIterator.hasNext(CheckedIndexedSeqView.scala:47)
	at scala.collection.StrictOptimizedIterableOps.map(StrictOptimizedIterableOps.scala:99)
	at scala.collection.StrictOptimizedIterableOps.map$(StrictOptimizedIterableOps.scala:87)
	at scala.collection.mutable.ArrayBuffer.map(ArrayBuffer.scala:43)
	at kafka.server.DelayedFetch.onComplete(DelayedFetch.scala:176)
	at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:72)
	at kafka.server.DelayedFetch.tryComplete(DelayedFetch.scala:145)
	at kafka.server.DelayedOperation.safeTryComplete(DelayedOperation.scala:121)
	at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:371)
	at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:264)
	at kafka.cluster.DelayedOperations$$anon$1.run(Partition.scala:74)
	at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
	at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
	at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
	at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
	at java.base/java.lang.Thread.run(Thread.java:833)
[2023-11-22 07:51:08,474] ERROR raAsyncOffset out of range, raAsyncOffset: 2537, startOffset: 2413, endOffset: 2537 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:08,475] ERROR raAsyncOffset out of range, raAsyncOffset: 3198, startOffset: 3074, endOffset: 3198 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:08,518] ERROR raAsyncOffset out of range, raAsyncOffset: 2548, startOffset: 2424, endOffset: 2548 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:08,856] ERROR raAsyncOffset out of range, raAsyncOffset: 2849, startOffset: 2725, endOffset: 2849 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:13,682] ERROR raAsyncOffset out of range, raAsyncOffset: 2410, startOffset: 2224, endOffset: 2410 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:13,958] ERROR raAsyncOffset out of range, raAsyncOffset: 2767, startOffset: 2538, endOffset: 2767 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:16,237] ERROR raAsyncOffset out of range, raAsyncOffset: 2530, startOffset: 2372, endOffset: 2530 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:16,237] ERROR raAsyncOffset out of range, raAsyncOffset: 2625, startOffset: 2523, endOffset: 2625 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:16,237] ERROR raAsyncOffset out of range, raAsyncOffset: 3198, startOffset: 3069, endOffset: 3198 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:17,468] ERROR raAsyncOffset out of range, raAsyncOffset: 3321, startOffset: 3284, endOffset: 3321 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:18,452] ERROR raAsyncOffset out of range, raAsyncOffset: 2969, startOffset: 2722, endOffset: 2969 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:18,452] ERROR raAsyncOffset out of range, raAsyncOffset: 3074, startOffset: 2953, endOffset: 3074 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:18,452] ERROR raAsyncOffset out of range, raAsyncOffset: 3090, startOffset: 2811, endOffset: 3090 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:19,200] ERROR raAsyncOffset out of range, raAsyncOffset: 3306, startOffset: 3096, endOffset: 3306 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:19,200] ERROR raAsyncOffset out of range, raAsyncOffset: 2621, startOffset: 2483, endOffset: 2621 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:19,200] ERROR raAsyncOffset out of range, raAsyncOffset: 2974, startOffset: 2787, endOffset: 2974 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:19,768] ERROR raAsyncOffset out of range, raAsyncOffset: 2571, startOffset: 2414, endOffset: 2571 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:19,768] ERROR raAsyncOffset out of range, raAsyncOffset: 2706, startOffset: 2339, endOffset: 2706 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:19,777] ERROR raAsyncOffset out of range, raAsyncOffset: 3027, startOffset: 2823, endOffset: 3027 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:19,778] ERROR raAsyncOffset out of range, raAsyncOffset: 2530, startOffset: 2411, endOffset: 2530 (com.automq.stream.s3.cache.BlockCache)
[2023-11-22 07:51:19,799] ERROR raAsyncOffset out of range, raAsyncOffset: 2533, startOffset: 2413, endOffset: 2533 (com.automq.stream.s3.cache.BlockCache)

[Chillax-0v0]

another case:

[2023-11-22 09:43:40,294] ERROR failed to complete delayed actions (kafka.server.ActionQueue)
scala.MatchError: null
	at kafka.server.DelayedFetch.$anonfun$onComplete$2(DelayedFetch.scala:176)
	at scala.collection.StrictOptimizedIterableOps.map(StrictOptimizedIterableOps.scala:100)
	at scala.collection.StrictOptimizedIterableOps.map$(StrictOptimizedIterableOps.scala:87)
	at scala.collection.mutable.ArrayBuffer.map(ArrayBuffer.scala:43)
	at kafka.server.DelayedFetch.onComplete(DelayedFetch.scala:176)
	at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:72)
	at kafka.server.DelayedFetch.tryComplete(DelayedFetch.scala:145)
	at kafka.server.DelayedOperation.safeTryComplete(DelayedOperation.scala:121)
	at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:371)
	at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:264)
	at kafka.server.ReplicaManager.$anonfun$appendRecords$6(ReplicaManager.scala:676)
	at scala.collection.mutable.HashMap$Node.foreach(HashMap.scala:627)
	at scala.collection.mutable.HashMap.foreach(HashMap.scala:489)
	at kafka.server.ReplicaManager.$anonfun$appendRecords$5(ReplicaManager.scala:665)
	at kafka.server.ActionQueue.tryCompleteActions(ActionQueue.scala:49)
	at kafka.server.ReplicaManager.tryCompleteActions(ReplicaManager.scala:623)
	at kafka.server.KafkaApis.handle(KafkaApis.scala:328)
	at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:77)
	at java.base/java.lang.Thread.run(Thread.java:833)
...
[2023-11-22 09:45:19,199] ERROR failed to complete delayed actions (kafka.server.ActionQueue)
scala.MatchError: null
	at kafka.server.DelayedFetch.$anonfun$onComplete$2(DelayedFetch.scala:176)
	at scala.collection.StrictOptimizedIterableOps.map(StrictOptimizedIterableOps.scala:100)
	at scala.collection.StrictOptimizedIterableOps.map$(StrictOptimizedIterableOps.scala:87)
	at scala.collection.mutable.ArrayBuffer.map(ArrayBuffer.scala:43)
	at kafka.server.DelayedFetch.onComplete(DelayedFetch.scala:176)
	at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:72)
	at kafka.server.DelayedFetch.tryComplete(DelayedFetch.scala:145)
	at kafka.server.DelayedOperation.safeTryComplete(DelayedOperation.scala:121)
	at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:371)
	at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:264)
	at kafka.server.ReplicaManager.$anonfun$appendRecords$6(ReplicaManager.scala:676)
	at scala.collection.mutable.HashMap$Node.foreach(HashMap.scala:627)
	at scala.collection.mutable.HashMap.foreach(HashMap.scala:489)
	at kafka.server.ReplicaManager.$anonfun$appendRecords$5(ReplicaManager.scala:665)
	at kafka.server.ActionQueue.tryCompleteActions(ActionQueue.scala:49)
	at kafka.server.ReplicaManager.tryCompleteActions(ReplicaManager.scala:623)
	at kafka.server.KafkaApis.handle(KafkaApis.scala:328)
	at kafka.server.KafkaRequestHandler.run(KafkaRequestHandler.scala:77)
	at java.base/java.lang.Thread.run(Thread.java:833)

[Chillax-0v0]

happened again...
version: 0.7.0 (Commit:361cafcb659f1736)

[2023-11-23 09:52:59,921] INFO [Stream id=387 epoch0] trim to 271 (com.automq.stream.s3.S3Stream)
[2023-11-23 09:52:59,921] INFO [Stream id=263 epoch0] trim to 271 (com.automq.stream.s3.S3Stream)
[2023-11-23 09:53:08,504] INFO log append cost, permitAcquireFail={count=0, max=0.0, mean=0.0}, remainingPermit=104791604/104857600, append={count=18772, max=58223.0, mean=912.3038568080119}, callback={count=18771, max=64540.0, mean=2387.5698151403762}>
[2023-11-23 09:53:08,514] INFO [KafkaApi-1] produce cost (in microseconds), produce={count=12187, max=534488.0, mean=2917.641175022565} callback={count=12185, max=569748.0, mean=4694.184653262208} ack={count=12185, max=45053.0, mean=227.37398440705786}>
[2023-11-23 09:53:09,332] INFO Reporting metrics. (kafka.autobalancer.metricsreporter.AutoBalancerMetricsReporter)
[2023-11-23 09:53:09,598] INFO Finished reporting metrics, total metrics size: 440, merged size: 181. (kafka.autobalancer.metricsreporter.AutoBalancerMetricsReporter)
[2023-11-23 09:53:10,399] ERROR delay fetch fail (kafka.cluster.DelayedOperations)
java.util.ConcurrentModificationException: mutation occurred during iteration
        at scala.collection.mutable.MutationTracker$.checkMutations(MutationTracker.scala:43)
        at scala.collection.mutable.CheckedIndexedSeqView$CheckedIterator.hasNext(CheckedIndexedSeqView.scala:47)
        at scala.collection.StrictOptimizedIterableOps.map(StrictOptimizedIterableOps.scala:99)
        at scala.collection.StrictOptimizedIterableOps.map$(StrictOptimizedIterableOps.scala:87)
        at scala.collection.mutable.ArrayBuffer.map(ArrayBuffer.scala:43)
        at kafka.server.DelayedFetch.onComplete(DelayedFetch.scala:176)
        at kafka.server.DelayedOperation.forceComplete(DelayedOperation.scala:72)
        at kafka.server.DelayedFetch.tryComplete(DelayedFetch.scala:145)
        at kafka.server.DelayedOperation.safeTryComplete(DelayedOperation.scala:121)
        at kafka.server.DelayedOperationPurgatory$Watchers.tryCompleteWatched(DelayedOperation.scala:371)
        at kafka.server.DelayedOperationPurgatory.checkAndComplete(DelayedOperation.scala:264)
        at kafka.cluster.DelayedOperations$$anon$1.run(Partition.scala:74)
        at java.base/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539)
        at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
        at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136)
        at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635)
        at java.base/java.lang.Thread.run(Thread.java:833)
[2023-11-23 09:53:19,628] INFO Reporting metrics. (kafka.autobalancer.metricsreporter.AutoBalancerMetricsReporter)
[2023-11-23 09:53:19,706] INFO Finished reporting metrics, total metrics size: 440, merged size: 181. (kafka.autobalancer.metricsreporter.AutoBalancerMetricsReporter)
[2023-11-23 09:53:20,547] INFO Upload delta WAL CommitStreamSetObjectRequest{objectId=0, orderId=0, objectSize=105153117, streamRanges=[(0--1,0-592), (1--1,0-195), (3--1,0-596), (4--1,0-596), (5--1,0-596), (6--1,0-596), (7--1,0-596), (8--1,0-596), (9-->
[2023-11-23 09:53:20,548] INFO try trim WAL to 119603771 (com.automq.stream.s3.S3Storage)
[2023-11-23 09:53:20,894] INFO Upload delta WAL CommitStreamSetObjectRequest{objectId=1, orderId=1, objectSize=105004645, streamRanges=[(15--1,502-745), (22--1,158-274), (34--1,317-564), (47--1,372-492), (48--1,226-454), (49--1,210-504), (50--1,473-556>
[2023-11-23 09:53:20,895] INFO try trim WAL to 234176735 (com.automq.stream.s3.S3Storage)
[2023-11-23 09:53:24,206] INFO Upload delta WAL CommitStreamSetObjectRequest{objectId=2, orderId=2, objectSize=105040547, streamRanges=[(15--1,745-885), (22--1,274-383), (34--1,564-643), (47--1,492-660), (48--1,454-785), (49--1,504-639), (50--1,556-924>
[2023-11-23 09:53:24,206] INFO try trim WAL to 350568644 (com.automq.stream.s3.S3Storage)
[2023-11-23 09:53:26,544] INFO Upload delta WAL CommitStreamSetObjectRequest{objectId=3, orderId=3, objectSize=105182697, streamRanges=[(15--1,885-1069), (22--1,383-505), (34--1,643-879), (47--1,660-840), (48--1,785-881), (49--1,639-1004), (50--1,924-1>
[2023-11-23 09:53:26,544] INFO try trim WAL to 471122152 (com.automq.stream.s3.S3Storage)
[2023-11-23 09:53:26,549] INFO Upload delta WAL CommitStreamSetObjectRequest{objectId=4, orderId=4, objectSize=105229830, streamRanges=[(15--1,1069-1179), (22--1,505-688), (34--1,879-1053), (47--1,840-1068), (48--1,881-1138), (49--1,1004-1357), (50--1,>
[2023-11-23 09:53:26,550] INFO try trim WAL to 591863989 (com.automq.stream.s3.S3Storage)
[2023-11-23 09:53:29,467] INFO Upload delta WAL CommitStreamSetObjectRequest{objectId=5, orderId=5, objectSize=105275924, streamRanges=[(1--1,195-376), (2--1,0-12), (15--1,1179-1508), (22--1,688-812), (34--1,1053-1377), (47--1,1068-1596), (48--1,1138-1>
[2023-11-23 09:53:29,467] INFO try trim WAL to 713474443 (com.automq.stream.s3.S3Storage)