You signed in with another tab or window. Reload to refresh your session.You signed out in another tab or window. Reload to refresh your session.You switched accounts on another tab or window. Reload to refresh your session.Dismiss alert
This issue has been automatically marked as "stale" because it has not had any activity for a while. It will be closed in 90 days if no further activity occurs. Thank you for your contributions.
System information
Describe the problem you're observing
when under highload, useing "zpool remove" to remove cache disk, the zfs will hang for ever.
Describe how to reproduce the problem
when under highload, useing "zpool remove" to remove cache disk, the zfs will hang for ever.
Include any warning/errors/backtraces from the system logs
root@node1:~# zpool status
pool: mypool-74
state: ONLINE
scan: none requested
config:
NAME STATE READ WRITE CKSUM
mypool-74 ONLINE 0 0 0
raidz1-0 ONLINE 0 0 0
sdf ONLINE 0 0 0
sdg ONLINE 0 0 0
sdh ONLINE 0 0 0
sdi ONLINE 0 0 0
sdj ONLINE 0 0 0
sdk ONLINE 0 0 0
sdl ONLINE 0 0 0
sdm ONLINE 0 0 0
sdn ONLINE 0 0 0
logs
74-w-0 ONLINE 0 0 0
74-w-1 ONLINE 0 0 0
74-w-2 ONLINE 0 0 0
74-w-3 ONLINE 0 0 0
74-w-4 ONLINE 0 0 0
74-w-5 ONLINE 0 0 0
74-w-6 ONLINE 0 0 0
74-w-7 ONLINE 0 0 0
cache
74-r-0 ONLINE 0 0 0
74-r-1 ONLINE 0 0 0
74-r-2 ONLINE 0 0 0
74-r-3 ONLINE 0 0 0
74-r-4 ONLINE 0 0 0
74-r-5 ONLINE 0 0 0
74-r-6 ONLINE 0 0 0
74-r-7 ONLINE 0 0 0
errors: No known data errors
root@node1:
2. zpool iostat 1
capacity operations bandwidth
pool alloc free read write read write
mypool-74 216G 9.60T 0 26.9K 0 724M
mypool-74 216G 9.60T 0 25.1K 0 655M
mypool-74 216G 9.60T 0 26.3K 0 727M
mypool-74 216G 9.60T 0 31.7K 3.98K 723M
mypool-74 222G 9.60T 1 31.2K 7.96K 753M
mypool-74 222G 9.60T 0 27.0K 0 689M
mypool-74 222G 9.60T 0 28.0K 0 704M
mypool-74 222G 9.60T 0 29.0K 0 710M
mypool-74 222G 9.60T 0 28.3K 3.98K 715M
mypool-74 222G 9.60T 0 28.1K 3.98K 747M
mypool-74 222G 9.60T 0 26.4K 0 735M
mypool-74 222G 9.60T 0 30.3K 0 794M
mypool-74 222G 9.60T 0 29.7K 0 829M
mypool-74 222G 9.60T 1 29.9K 7.96K 800M
mypool-74 222G 9.60T 0 29.2K 3.98K 777M
mypool-74 222G 9.60T 0 29.8K 0 816M
mypool-74 222G 9.60T 1 21.5K 7.96K 569M
mypool-74 228G 9.59T 0 29.9K 0 661M
mypool-74 228G 9.59T 0 29.9K 3.98K 762M
mypool-74 228G 9.59T 1 30.3K 7.96K 795M
mypool-74 228G 9.59T 0 30.8K 3.98K 784M
mypool-74 228G 9.59T 0 30.5K 3.98K 809M
mypool-74 228G 9.59T 0 29.8K 0 758M
mypool-74 228G 9.59T 0 31.0K 0 813M
3. zpool remove mypool-74 /dev/mapper/74-r-0 /dev/mapper/74-r-1 /dev/mapper/74-r-2 /dev/mapper/74-r-3 /dev/mapper/74-r-4 /dev/mapper/74-r-5 /dev/mapper/74-r-6 /dev/mapper/74-r-7
will hang for ever and all zfs command will hang
4. ps -aux | grep D | grep -v kworker
USER PID %CPU %MEM VSZ RSS TTY STAT START TIME COMMAND
root 2004 0.2 0.0 0 0 ? D 16:29 0:09 [l2arc_feed]
root 3480 0.0 0.0 61380 5236 ? Ss 16:29 0:00 /usr/sbin/sshd -D
haproxy 3723 0.0 0.0 29000 772 ? Ss 16:29 0:00 /usr/sbin/haproxy -f /etc/haproxy/haproxy.cfg -D -p /var/run/haproxy.pid
root 5594 3.1 0.0 0 0 ? D 16:35 2:02 [md8_resync]
root 6571 0.0 0.0 0 0 ? D 16:36 0:01 [txg_sync]
root 6572 0.0 0.0 0 0 ? D 16:36 0:00 [mmp]
root 17420 0.0 0.0 11752 2232 pts/3 S+ 17:40 0:00 grep --color=auto D
root 19673 0.0 0.0 37064 5260 pts/5 D+ 17:00 0:00 zpool iostat 1
root 30032 0.0 0.0 36276 4344 pts/4 D+ 17:03 0:00 zpool remove mypool-74 /dev/mapper/74-r-0 /dev/mapper/74-r-1 /dev/mapper/74-r-2 /dev/mapper/74-r-3 /dev/mapper/74-r-4 /dev/mapper/74-r-5 /dev/mapper/74-r-6 /dev/mapper/74-r-7
5. syslog
Mar 23 17:02:21 node1 kernel: [ 2013.428707] perf interrupt took too long (2514 > 2500), lowering kernel.perf_event_max_sample_rate to 50000
Mar 23 17:05:01 node1 CRON[16762]: (root) CMD (command -v debian-sa1 > /dev/null && debian-sa1 1 1)
Mar 23 17:05:16 node1 kernel: [ 2188.841553] ABORT_TASK: Found referenced qla2xxx task_tag: 1214176, udev_path: /dev/zd48, cdb[0]: 0x2a
Mar 23 17:05:16 node1 kernel: [ 2188.842571] ABORT_TASK: Found referenced qla2xxx task_tag: 1214944, udev_path: /dev/zd0, cdb[0]: 0x2a
Mar 23 17:05:16 node1 kernel: [ 2188.843800] ABORT_TASK: Found referenced qla2xxx task_tag: 1215712, udev_path: /dev/zd32, cdb[0]: 0x2a
Mar 23 17:05:17 node1 kernel: [ 2188.941676] ABORT_TASK: Found referenced qla2xxx task_tag: 1216480, udev_path: /dev/zd16, cdb[0]: 0x2a
Mar 23 17:06:48 node1 kernel: [ 2280.632492] INFO: task kworker/10:0:61 blocked for more than 120 seconds.
Mar 23 17:06:48 node1 kernel: [ 2280.632495] Tainted: P OE 4.4.0-24-generic #43
14.04.114.04.1Mar 23 17:06:48 node1 kernel: [ 2280.632496] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Mar 23 17:06:48 node1 kernel: [ 2280.632498] kworker/10:0 D ffff8820387c7978 0 61 2 0x00000000
Mar 23 17:06:48 node1 kernel: [ 2280.632507] Workqueue: tcm_qla2xxx_free tcm_qla2xxx_handle_data_work [tcm_qla2xxx]
Mar 23 17:06:48 node1 kernel: [ 2280.632508] ffff8820387c7978 ffff881038e89b80 ffff88203877c4c0 ffff8820387c8000
Mar 23 17:06:48 node1 kernel: [ 2280.632510] ffff88203827690c ffff88203877c4c0 00000000ffffffff ffff882038276910
Mar 23 17:06:48 node1 kernel: [ 2280.632512] ffff8820387c7990 ffffffff817ed145 ffff882038276908 ffff8820387c79a0
Mar 23 17:06:48 node1 kernel: [ 2280.632513] Call Trace:
Mar 23 17:06:48 node1 kernel: [ 2280.632522] [] schedule+0x35/0x80
Mar 23 17:06:48 node1 kernel: [ 2280.632524] [] schedule_preempt_disabled+0xe/0x10
Mar 23 17:06:48 node1 kernel: [ 2280.632528] [] __mutex_lock_slowpath+0x95/0x110
Mar 23 17:06:48 node1 kernel: [ 2280.632530] [] mutex_lock+0x1f/0x2f
Mar 23 17:06:48 node1 kernel: [ 2280.632595] [] zil_commit_impl+0x136/0xc90 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.632603] [] ? spl_kmem_free+0x2a/0x40 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.632625] [] ? aggsum_add+0x17c/0x1a0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.632664] [] zil_commit+0x3d/0x60 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.632703] [] zvol_write+0x508/0x550 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.632741] [] zvol_request+0x24c/0x3f0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.632745] [] generic_make_request+0xe1/0x1a0
Mar 23 17:06:48 node1 kernel: [ 2280.632747] [] submit_bio+0x77/0x150
Mar 23 17:06:48 node1 kernel: [ 2280.632750] [] ? bvec_alloc+0x5c/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.632754] [] iblock_submit_bios+0x3b/0x60 [target_core_iblock]
Mar 23 17:06:48 node1 kernel: [ 2280.632756] [] iblock_execute_rw+0x1a2/0x410 [target_core_iblock]
Mar 23 17:06:48 node1 kernel: [ 2280.632769] [] sbc_execute_rw+0x22/0x30 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.632775] [] __target_execute_cmd+0x36/0xd0 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.632780] [] ? __switch_to+0x1d6/0x570
Mar 23 17:06:48 node1 kernel: [ 2280.632785] [] target_execute_cmd+0x10c/0x2e0 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.632787] [] tcm_qla2xxx_handle_data_work+0xb7/0xe0 [tcm_qla2xxx]
Mar 23 17:06:48 node1 kernel: [ 2280.632793] [] process_one_work+0x150/0x3f0
Mar 23 17:06:48 node1 kernel: [ 2280.632795] [] worker_thread+0x11a/0x470
Mar 23 17:06:48 node1 kernel: [ 2280.632797] [] ? rescuer_thread+0x310/0x310
Mar 23 17:06:48 node1 kernel: [ 2280.632799] [] kthread+0xd2/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.632800] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.632804] [] ret_from_fork+0x3f/0x70
Mar 23 17:06:48 node1 kernel: [ 2280.632805] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.632830] INFO: task kworker/9:1:421 blocked for more than 120 seconds.
Mar 23 17:06:48 node1 kernel: [ 2280.632831] Tainted: P OE 4.4.0-24-generic #43
Mar 23 17:06:48 node1 kernel: [ 2280.632832] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Mar 23 17:06:48 node1 kernel: [ 2280.632833] kworker/9:1 D ffff88203465f978 0 421 2 0x00000000
Mar 23 17:06:48 node1 kernel: [ 2280.632837] Workqueue: tcm_qla2xxx_free tcm_qla2xxx_handle_data_work [tcm_qla2xxx]
Mar 23 17:06:48 node1 kernel: [ 2280.632837] ffff88203465f978 ffff881038e88dc0 ffff882034668000 ffff882034660000
Mar 23 17:06:48 node1 kernel: [ 2280.632839] ffff881012bb390c ffff882034668000 00000000ffffffff ffff881012bb3910
Mar 23 17:06:48 node1 kernel: [ 2280.632841] ffff88203465f990 ffffffff817ed145 ffff881012bb3908 ffff88203465f9a0
Mar 23 17:06:48 node1 kernel: [ 2280.632842] Call Trace:
Mar 23 17:06:48 node1 kernel: [ 2280.632844] [] schedule+0x35/0x80
Mar 23 17:06:48 node1 kernel: [ 2280.632846] [] schedule_preempt_disabled+0xe/0x10
Mar 23 17:06:48 node1 kernel: [ 2280.632848] [] __mutex_lock_slowpath+0x95/0x110
Mar 23 17:06:48 node1 kernel: [ 2280.632850] [] mutex_lock+0x1f/0x2f
Mar 23 17:06:48 node1 kernel: [ 2280.632888] [] zil_commit_impl+0x136/0xc90 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.632892] [] ? spl_kmem_free+0x2a/0x40 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.632913] [] ? aggsum_add+0x17c/0x1a0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.632951] [] zil_commit+0x3d/0x60 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.632988] [] zvol_write+0x508/0x550 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633026] [] zvol_request+0x24c/0x3f0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633028] [] generic_make_request+0xe1/0x1a0
Mar 23 17:06:48 node1 kernel: [ 2280.633029] [] submit_bio+0x77/0x150
Mar 23 17:06:48 node1 kernel: [ 2280.633031] [] ? bvec_alloc+0x5c/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.633033] [] iblock_submit_bios+0x3b/0x60 [target_core_iblock]
Mar 23 17:06:48 node1 kernel: [ 2280.633035] [] iblock_execute_rw+0x1a2/0x410 [target_core_iblock]
Mar 23 17:06:48 node1 kernel: [ 2280.633042] [] sbc_execute_rw+0x22/0x30 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.633048] [] __target_execute_cmd+0x36/0xd0 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.633049] [] ? __switch_to+0x1d6/0x570
Mar 23 17:06:48 node1 kernel: [ 2280.633055] [] target_execute_cmd+0x10c/0x2e0 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.633057] [] tcm_qla2xxx_handle_data_work+0xb7/0xe0 [tcm_qla2xxx]
Mar 23 17:06:48 node1 kernel: [ 2280.633059] [] process_one_work+0x150/0x3f0
Mar 23 17:06:48 node1 kernel: [ 2280.633061] [] worker_thread+0x11a/0x470
Mar 23 17:06:48 node1 kernel: [ 2280.633063] [] ? rescuer_thread+0x310/0x310
Mar 23 17:06:48 node1 kernel: [ 2280.633064] [] kthread+0xd2/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.633066] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.633067] [] ret_from_fork+0x3f/0x70
Mar 23 17:06:48 node1 kernel: [ 2280.633069] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.633070] INFO: task kworker/10:1:422 blocked for more than 120 seconds.
Mar 23 17:06:48 node1 kernel: [ 2280.633071] Tainted: P OE 4.4.0-24-generic #43
14.04.114.04.1Mar 23 17:06:48 node1 kernel: [ 2280.633072] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Mar 23 17:06:48 node1 kernel: [ 2280.633073] kworker/10:1 D ffff882034673978 0 422 2 0x00000000
Mar 23 17:06:48 node1 kernel: [ 2280.633076] Workqueue: tcm_qla2xxx_free tcm_qla2xxx_handle_data_work [tcm_qla2xxx]
Mar 23 17:06:48 node1 kernel: [ 2280.633077] ffff882034673978 ffff881038e89b80 ffff882034668dc0 ffff882034674000
Mar 23 17:06:48 node1 kernel: [ 2280.633078] ffff88203827690c ffff882034668dc0 00000000ffffffff ffff882038276910
Mar 23 17:06:48 node1 kernel: [ 2280.633079] ffff882034673990 ffffffff817ed145 ffff882038276908 ffff8820346739a0
Mar 23 17:06:48 node1 kernel: [ 2280.633081] Call Trace:
Mar 23 17:06:48 node1 kernel: [ 2280.633083] [] schedule+0x35/0x80
Mar 23 17:06:48 node1 kernel: [ 2280.633084] [] schedule_preempt_disabled+0xe/0x10
Mar 23 17:06:48 node1 kernel: [ 2280.633086] [] __mutex_lock_slowpath+0x95/0x110
Mar 23 17:06:48 node1 kernel: [ 2280.633088] [] mutex_lock+0x1f/0x2f
Mar 23 17:06:48 node1 kernel: [ 2280.633125] [] zil_commit_impl+0x136/0xc90 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633129] [] ? spl_kmem_free+0x2a/0x40 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.633150] [] ? aggsum_add+0x17c/0x1a0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633187] [] zil_commit+0x3d/0x60 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633224] [] zvol_write+0x508/0x550 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633261] [] zvol_request+0x24c/0x3f0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633263] [] generic_make_request+0xe1/0x1a0
Mar 23 17:06:48 node1 kernel: [ 2280.633264] [] submit_bio+0x77/0x150
Mar 23 17:06:48 node1 kernel: [ 2280.633266] [] ? bvec_alloc+0x5c/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.633268] [] iblock_submit_bios+0x3b/0x60 [target_core_iblock]
Mar 23 17:06:48 node1 kernel: [ 2280.633269] [] iblock_execute_rw+0x1a2/0x410 [target_core_iblock]
Mar 23 17:06:48 node1 kernel: [ 2280.633277] [] sbc_execute_rw+0x22/0x30 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.633283] [] __target_execute_cmd+0x36/0xd0 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.633284] [] ? __switch_to+0x1d6/0x570
Mar 23 17:06:48 node1 kernel: [ 2280.633290] [] target_execute_cmd+0x10c/0x2e0 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.633292] [] tcm_qla2xxx_handle_data_work+0xb7/0xe0 [tcm_qla2xxx]
Mar 23 17:06:48 node1 kernel: [ 2280.633294] [] process_one_work+0x150/0x3f0
Mar 23 17:06:48 node1 kernel: [ 2280.633296] [] worker_thread+0x11a/0x470
Mar 23 17:06:48 node1 kernel: [ 2280.633298] [] ? __schedule+0x359/0x980
Mar 23 17:06:48 node1 kernel: [ 2280.633299] [] ? rescuer_thread+0x310/0x310
Mar 23 17:06:48 node1 kernel: [ 2280.633301] [] kthread+0xd2/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.633302] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.633304] [] ret_from_fork+0x3f/0x70
Mar 23 17:06:48 node1 kernel: [ 2280.633305] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.633320] INFO: task l2arc_feed:2004 blocked for more than 120 seconds.
Mar 23 17:06:48 node1 kernel: [ 2280.633321] Tainted: P OE 4.4.0-24-generic #43
Mar 23 17:06:48 node1 kernel: [ 2280.633321] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Mar 23 17:06:48 node1 kernel: [ 2280.633322] l2arc_feed D ffff88101ed3bca8 0 2004 2 0x00000000
Mar 23 17:06:48 node1 kernel: [ 2280.633324] ffff88101ed3bca8 ffff881038eb0dc0 ffff88102dab3700 ffff88101ed3c000
Mar 23 17:06:48 node1 kernel: [ 2280.633326] ffffffffc1393764 ffff88102dab3700 00000000ffffffff ffffffffc1393768
Mar 23 17:06:48 node1 kernel: [ 2280.633327] ffff88101ed3bcc0 ffffffff817ed145 ffffffffc1393760 ffff88101ed3bcd0
Mar 23 17:06:48 node1 kernel: [ 2280.633329] Call Trace:
Mar 23 17:06:48 node1 kernel: [ 2280.633331] [] schedule+0x35/0x80
Mar 23 17:06:48 node1 kernel: [ 2280.633333] [] schedule_preempt_disabled+0xe/0x10
Mar 23 17:06:48 node1 kernel: [ 2280.633335] [] __mutex_lock_slowpath+0x95/0x110
Mar 23 17:06:48 node1 kernel: [ 2280.633339] [] ? prepare_to_wait_event+0xf0/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.633341] [] mutex_lock+0x1f/0x2f
Mar 23 17:06:48 node1 kernel: [ 2280.633364] [] l2arc_feed_thread+0x1cc/0x10a0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633367] [] ? check_preempt_curr+0x75/0x90
Mar 23 17:06:48 node1 kernel: [ 2280.633390] [] ? l2arc_evict+0x2a0/0x2a0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633395] [] thread_generic_wrapper+0x7a/0x90 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.633398] [] ? __thread_exit+0x20/0x20 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.633400] [] kthread+0xd2/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.633401] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.633402] [] ret_from_fork+0x3f/0x70
Mar 23 17:06:48 node1 kernel: [ 2280.633403] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.633425] INFO: task kworker/9:2:4934 blocked for more than 120 seconds.
Mar 23 17:06:48 node1 kernel: [ 2280.633426] Tainted: P OE 4.4.0-24-generic #43
14.04.114.04.1Mar 23 17:06:48 node1 kernel: [ 2280.633427] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Mar 23 17:06:48 node1 kernel: [ 2280.633427] kworker/9:2 D ffff88202649f978 0 4934 2 0x00000000
Mar 23 17:06:48 node1 kernel: [ 2280.633431] Workqueue: tcm_qla2xxx_free tcm_qla2xxx_handle_data_work [tcm_qla2xxx]
Mar 23 17:06:48 node1 kernel: [ 2280.633432] ffff88202649f978 ffff881038e88dc0 ffff882028180dc0 ffff8820264a0000
Mar 23 17:06:48 node1 kernel: [ 2280.633433] ffff88103330e10c ffff882028180dc0 00000000ffffffff ffff88103330e110
Mar 23 17:06:48 node1 kernel: [ 2280.633435] ffff88202649f990 ffffffff817ed145 ffff88103330e108 ffff88202649f9a0
Mar 23 17:06:48 node1 kernel: [ 2280.633436] Call Trace:
Mar 23 17:06:48 node1 kernel: [ 2280.633438] [] schedule+0x35/0x80
Mar 23 17:06:48 node1 kernel: [ 2280.633440] [] schedule_preempt_disabled+0xe/0x10
Mar 23 17:06:48 node1 kernel: [ 2280.633442] [] __mutex_lock_slowpath+0x95/0x110
Mar 23 17:06:48 node1 kernel: [ 2280.633444] [] mutex_lock+0x1f/0x2f
Mar 23 17:06:48 node1 kernel: [ 2280.633481] [] zil_commit_impl+0x136/0xc90 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633485] [] ? spl_kmem_free+0x2a/0x40 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.633507] [] ? aggsum_add+0x17c/0x1a0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633544] [] zil_commit+0x3d/0x60 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633581] [] zvol_write+0x508/0x550 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633618] [] zvol_request+0x24c/0x3f0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633620] [] generic_make_request+0xe1/0x1a0
Mar 23 17:06:48 node1 kernel: [ 2280.633621] [] submit_bio+0x77/0x150
Mar 23 17:06:48 node1 kernel: [ 2280.633623] [] ? bvec_alloc+0x5c/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.633625] [] iblock_submit_bios+0x3b/0x60 [target_core_iblock]
Mar 23 17:06:48 node1 kernel: [ 2280.633627] [] iblock_execute_rw+0x1a2/0x410 [target_core_iblock]
Mar 23 17:06:48 node1 kernel: [ 2280.633634] [] sbc_execute_rw+0x22/0x30 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.633640] [] __target_execute_cmd+0x36/0xd0 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.633641] [] ? __switch_to+0x1d6/0x570
Mar 23 17:06:48 node1 kernel: [ 2280.633647] [] target_execute_cmd+0x10c/0x2e0 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.633649] [] tcm_qla2xxx_handle_data_work+0xb7/0xe0 [tcm_qla2xxx]
Mar 23 17:06:48 node1 kernel: [ 2280.633651] [] process_one_work+0x150/0x3f0
Mar 23 17:06:48 node1 kernel: [ 2280.633653] [] worker_thread+0x11a/0x470
Mar 23 17:06:48 node1 kernel: [ 2280.633655] [] ? __schedule+0x359/0x980
Mar 23 17:06:48 node1 kernel: [ 2280.633656] [] ? rescuer_thread+0x310/0x310
Mar 23 17:06:48 node1 kernel: [ 2280.633657] [] kthread+0xd2/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.633659] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.633660] [] ret_from_fork+0x3f/0x70
Mar 23 17:06:48 node1 kernel: [ 2280.633661] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.633688] INFO: task txg_sync:6571 blocked for more than 120 seconds.
Mar 23 17:06:48 node1 kernel: [ 2280.633689] Tainted: P OE 4.4.0-24-generic #43
Mar 23 17:06:48 node1 kernel: [ 2280.633690] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Mar 23 17:06:48 node1 kernel: [ 2280.633690] txg_sync D ffff882021bd3c90 0 6571 2 0x00000000
Mar 23 17:06:48 node1 kernel: [ 2280.633692] ffff882021bd3c90 ffff881038c39b80 ffff882021a844c0 ffff882021bd4000
Mar 23 17:06:48 node1 kernel: [ 2280.633693] ffff880036079ec0 ffff880036079f10 ffff880036079ee8 0000000000000000
Mar 23 17:06:48 node1 kernel: [ 2280.633695] ffff882021bd3ca8 ffffffff817ed145 ffff880036079f08 ffff882021bd3d08
Mar 23 17:06:48 node1 kernel: [ 2280.633696] Call Trace:
Mar 23 17:06:48 node1 kernel: [ 2280.633698] [] schedule+0x35/0x80
Mar 23 17:06:48 node1 kernel: [ 2280.633701] [] cv_wait_common+0xed/0x120 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.633703] [] ? prepare_to_wait_event+0xf0/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.633706] [] __cv_wait+0x15/0x20 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.633741] [] spa_config_enter+0xf8/0x110 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633776] [] spa_txg_history_fini_io+0x41/0x200 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633811] [] txg_sync_thread+0x2bf/0x470 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633845] [] ? txg_quiesce_thread+0x390/0x390 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633850] [] thread_generic_wrapper+0x7a/0x90 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.633853] [] ? __thread_exit+0x20/0x20 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.633854] [] kthread+0xd2/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.633856] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.633857] [] ret_from_fork+0x3f/0x70
Mar 23 17:06:48 node1 kernel: [ 2280.633858] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.633860] INFO: task mmp:6572 blocked for more than 120 seconds.
Mar 23 17:06:48 node1 kernel: [ 2280.633861] Tainted: P OE 4.4.0-24-generic #43
14.04.114.04.1Mar 23 17:06:48 node1 kernel: [ 2280.633861] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Mar 23 17:06:48 node1 kernel: [ 2280.633862] mmp D ffff882021bd7ca8 0 6572 2 0x00000000
Mar 23 17:06:48 node1 kernel: [ 2280.633863] ffff882021bd7ca8 ffff881038e8d280 ffff882021a85280 ffff882021bd8000
Mar 23 17:06:48 node1 kernel: [ 2280.633865] ffff88003607a250 ffff88003607a2a0 ffff88003607a278 0000000000000000
Mar 23 17:06:48 node1 kernel: [ 2280.633866] ffff882021bd7cc0 ffffffff817ed145 ffff88003607a298 ffff882021bd7d20
Mar 23 17:06:48 node1 kernel: [ 2280.633868] Call Trace:
Mar 23 17:06:48 node1 kernel: [ 2280.633870] [] schedule+0x35/0x80
Mar 23 17:06:48 node1 kernel: [ 2280.633872] [] cv_wait_common+0xed/0x120 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.633874] [] ? prepare_to_wait_event+0xf0/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.633877] [] __cv_wait+0x15/0x20 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.633911] [] spa_config_enter+0xf8/0x110 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633945] [] vdev_count_leaves+0x25/0x50 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633977] [] mmp_thread+0x33f/0x700 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.633979] [] ? __slab_free+0x66/0x2a0
Mar 23 17:06:48 node1 kernel: [ 2280.633981] [] ? put_prev_entity+0x35/0x670
Mar 23 17:06:48 node1 kernel: [ 2280.634012] [] ? mmp_write_uberblock+0x6d0/0x6d0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.634016] [] thread_generic_wrapper+0x7a/0x90 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.634020] [] ? __thread_exit+0x20/0x20 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.634021] [] kthread+0xd2/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.634022] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.634024] [] ret_from_fork+0x3f/0x70
Mar 23 17:06:48 node1 kernel: [ 2280.634025] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.634030] INFO: task kworker/10:2:7889 blocked for more than 120 seconds.
Mar 23 17:06:48 node1 kernel: [ 2280.634031] Tainted: P OE 4.4.0-24-generic #43
Mar 23 17:06:48 node1 kernel: [ 2280.634032] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Mar 23 17:06:48 node1 kernel: [ 2280.634032] kworker/10:2 D ffff882028267978 0 7889 2 0x00000000
Mar 23 17:06:48 node1 kernel: [ 2280.634036] Workqueue: tcm_qla2xxx_free tcm_qla2xxx_handle_data_work [tcm_qla2xxx]
Mar 23 17:06:48 node1 kernel: [ 2280.634036] ffff882028267978 ffff881038e89b80 ffff88202ad72940 ffff882028268000
Mar 23 17:06:48 node1 kernel: [ 2280.634038] ffff88203827690c ffff88202ad72940 00000000ffffffff ffff882038276910
Mar 23 17:06:48 node1 kernel: [ 2280.634039] ffff882028267990 ffffffff817ed145 ffff882038276908 ffff8820282679a0
Mar 23 17:06:48 node1 kernel: [ 2280.634041] Call Trace:
Mar 23 17:06:48 node1 kernel: [ 2280.634043] [] schedule+0x35/0x80
Mar 23 17:06:48 node1 kernel: [ 2280.634045] [] schedule_preempt_disabled+0xe/0x10
Mar 23 17:06:48 node1 kernel: [ 2280.634047] [] __mutex_lock_slowpath+0x95/0x110
Mar 23 17:06:48 node1 kernel: [ 2280.634048] [] mutex_lock+0x1f/0x2f
Mar 23 17:06:48 node1 kernel: [ 2280.634087] [] zil_commit_impl+0x136/0xc90 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.634091] [] ? spl_kmem_free+0x2a/0x40 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.634112] [] ? aggsum_add+0x17c/0x1a0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.634151] [] zil_commit+0x3d/0x60 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.634189] [] zvol_write+0x508/0x550 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.634227] [] zvol_request+0x24c/0x3f0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.634229] [] generic_make_request+0xe1/0x1a0
Mar 23 17:06:48 node1 kernel: [ 2280.634230] [] submit_bio+0x77/0x150
Mar 23 17:06:48 node1 kernel: [ 2280.634232] [] ? bvec_alloc+0x5c/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.634234] [] iblock_submit_bios+0x3b/0x60 [target_core_iblock]
Mar 23 17:06:48 node1 kernel: [ 2280.634235] [] iblock_execute_rw+0x1a2/0x410 [target_core_iblock]
Mar 23 17:06:48 node1 kernel: [ 2280.634243] [] sbc_execute_rw+0x22/0x30 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.634248] [] __target_execute_cmd+0x36/0xd0 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.634250] [] ? __switch_to+0x1d6/0x570
Mar 23 17:06:48 node1 kernel: [ 2280.634256] [] target_execute_cmd+0x10c/0x2e0 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.634258] [] tcm_qla2xxx_handle_data_work+0xb7/0xe0 [tcm_qla2xxx]
Mar 23 17:06:48 node1 kernel: [ 2280.634260] [] process_one_work+0x150/0x3f0
Mar 23 17:06:48 node1 kernel: [ 2280.634262] [] worker_thread+0x11a/0x470
Mar 23 17:06:48 node1 kernel: [ 2280.634264] [] ? rescuer_thread+0x310/0x310
Mar 23 17:06:48 node1 kernel: [ 2280.634265] [] kthread+0xd2/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.634266] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.634268] [] ret_from_fork+0x3f/0x70
Mar 23 17:06:48 node1 kernel: [ 2280.634269] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.634271] INFO: task kworker/10:4:7907 blocked for more than 120 seconds.
Mar 23 17:06:48 node1 kernel: [ 2280.634272] Tainted: P OE 4.4.0-24-generic #43
14.04.114.04.1Mar 23 17:06:48 node1 kernel: [ 2280.634272] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Mar 23 17:06:48 node1 kernel: [ 2280.634273] kworker/10:4 D ffff882023a7b978 0 7907 2 0x00000000
Mar 23 17:06:48 node1 kernel: [ 2280.634276] Workqueue: tcm_qla2xxx_free tcm_qla2xxx_handle_data_work [tcm_qla2xxx]
Mar 23 17:06:48 node1 kernel: [ 2280.634277] ffff882023a7b978 ffff881038e89b80 ffff8820350c8dc0 ffff882023a7c000
Mar 23 17:06:48 node1 kernel: [ 2280.634278] ffff88203827690c ffff8820350c8dc0 00000000ffffffff ffff882038276910
Mar 23 17:06:48 node1 kernel: [ 2280.634279] ffff882023a7b990 ffffffff817ed145 ffff882038276908 ffff882023a7b9a0
Mar 23 17:06:48 node1 kernel: [ 2280.634281] Call Trace:
Mar 23 17:06:48 node1 kernel: [ 2280.634282] [] schedule+0x35/0x80
Mar 23 17:06:48 node1 kernel: [ 2280.634284] [] schedule_preempt_disabled+0xe/0x10
Mar 23 17:06:48 node1 kernel: [ 2280.634286] [] __mutex_lock_slowpath+0x95/0x110
Mar 23 17:06:48 node1 kernel: [ 2280.634288] [] mutex_lock+0x1f/0x2f
Mar 23 17:06:48 node1 kernel: [ 2280.634325] [] zil_commit_impl+0x136/0xc90 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.634329] [] ? spl_kmem_free+0x2a/0x40 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.634351] [] ? aggsum_add+0x17c/0x1a0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.634388] [] zil_commit+0x3d/0x60 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.634426] [] zvol_write+0x508/0x550 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.634463] [] zvol_request+0x24c/0x3f0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.634465] [] generic_make_request+0xe1/0x1a0
Mar 23 17:06:48 node1 kernel: [ 2280.634466] [] submit_bio+0x77/0x150
Mar 23 17:06:48 node1 kernel: [ 2280.634468] [] ? bvec_alloc+0x5c/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.634470] [] iblock_submit_bios+0x3b/0x60 [target_core_iblock]
Mar 23 17:06:48 node1 kernel: [ 2280.634471] [] iblock_execute_rw+0x1a2/0x410 [target_core_iblock]
Mar 23 17:06:48 node1 kernel: [ 2280.634478] [] sbc_execute_rw+0x22/0x30 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.634484] [] __target_execute_cmd+0x36/0xd0 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.634486] [] ? __switch_to+0x1d6/0x570
Mar 23 17:06:48 node1 kernel: [ 2280.634492] [] target_execute_cmd+0x10c/0x2e0 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.634494] [] tcm_qla2xxx_handle_data_work+0xb7/0xe0 [tcm_qla2xxx]
Mar 23 17:06:48 node1 kernel: [ 2280.634496] [] process_one_work+0x150/0x3f0
Mar 23 17:06:48 node1 kernel: [ 2280.634497] [] worker_thread+0x11a/0x470
Mar 23 17:06:48 node1 kernel: [ 2280.634499] [] ? rescuer_thread+0x310/0x310
Mar 23 17:06:48 node1 kernel: [ 2280.634501] [] kthread+0xd2/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.634502] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.634504] [] ret_from_fork+0x3f/0x70
Mar 23 17:06:48 node1 kernel: [ 2280.634505] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.634506] INFO: task kworker/10:5:7922 blocked for more than 120 seconds.
Mar 23 17:06:48 node1 kernel: [ 2280.634507] Tainted: P OE 4.4.0-24-generic #43
Mar 23 17:06:48 node1 kernel: [ 2280.634508] "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Mar 23 17:06:48 node1 kernel: [ 2280.634508] kworker/10:5 D ffff88201c283978 0 7922 2 0x00000000
Mar 23 17:06:48 node1 kernel: [ 2280.634511] Workqueue: tcm_qla2xxx_free tcm_qla2xxx_handle_data_work [tcm_qla2xxx]
Mar 23 17:06:48 node1 kernel: [ 2280.634512] ffff88201c283978 ffff881038e89b80 ffff882038778000 ffff88201c284000
Mar 23 17:06:48 node1 kernel: [ 2280.634513] ffff88103330910c ffff882038778000 00000000ffffffff ffff881033309110
Mar 23 17:06:48 node1 kernel: [ 2280.634515] ffff88201c283990 ffffffff817ed145 ffff881033309108 ffff88201c2839a0
Mar 23 17:06:48 node1 kernel: [ 2280.634516] Call Trace:
Mar 23 17:06:48 node1 kernel: [ 2280.634518] [] schedule+0x35/0x80
Mar 23 17:06:48 node1 kernel: [ 2280.634519] [] schedule_preempt_disabled+0xe/0x10
Mar 23 17:06:48 node1 kernel: [ 2280.634521] [] __mutex_lock_slowpath+0x95/0x110
Mar 23 17:06:48 node1 kernel: [ 2280.634523] [] mutex_lock+0x1f/0x2f
Mar 23 17:06:48 node1 kernel: [ 2280.634560] [] zil_commit_impl+0x136/0xc90 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.634564] [] ? spl_kmem_free+0x2a/0x40 [spl]
Mar 23 17:06:48 node1 kernel: [ 2280.634585] [] ? aggsum_add+0x17c/0x1a0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.634622] [] zil_commit+0x3d/0x60 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.634659] [] zvol_write+0x508/0x550 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.634696] [] zvol_request+0x24c/0x3f0 [zfs]
Mar 23 17:06:48 node1 kernel: [ 2280.634698] [] generic_make_request+0xe1/0x1a0
Mar 23 17:06:48 node1 kernel: [ 2280.634700] [] submit_bio+0x77/0x150
Mar 23 17:06:48 node1 kernel: [ 2280.634701] [] ? bvec_alloc+0x5c/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.634703] [] iblock_submit_bios+0x3b/0x60 [target_core_iblock]
Mar 23 17:06:48 node1 kernel: [ 2280.634705] [] iblock_execute_rw+0x1a2/0x410 [target_core_iblock]
Mar 23 17:06:48 node1 kernel: [ 2280.634712] [] sbc_execute_rw+0x22/0x30 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.634718] [] __target_execute_cmd+0x36/0xd0 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.634719] [] ? __switch_to+0x1d6/0x570
Mar 23 17:06:48 node1 kernel: [ 2280.634725] [] target_execute_cmd+0x10c/0x2e0 [target_core_mod]
Mar 23 17:06:48 node1 kernel: [ 2280.634727] [] tcm_qla2xxx_handle_data_work+0xb7/0xe0 [tcm_qla2xxx]
Mar 23 17:06:48 node1 kernel: [ 2280.634729] [] process_one_work+0x150/0x3f0
Mar 23 17:06:48 node1 kernel: [ 2280.634731] [] worker_thread+0x11a/0x470
Mar 23 17:06:48 node1 kernel: [ 2280.634732] [] ? rescuer_thread+0x310/0x310
Mar 23 17:06:48 node1 kernel: [ 2280.634734] [] kthread+0xd2/0xf0
Mar 23 17:06:48 node1 kernel: [ 2280.634735] [] ? kthread_park+0x50/0x50
Mar 23 17:06:48 node1 kernel: [ 2280.634737] [] ret_from_fork+0x3f/0x70
Mar 23 17:06:48 node1 kernel: [ 2280.634738] [] ? kthread_park+0x50/0x50
Mar 23 17:09:01 node1 CRON[16946]: (root) CMD ( [ -x /usr/lib/php5/maxlifetime ] && [ -x /usr/lib/php5/sessionclean ] && [ -d /var/lib/php5 ] && /usr/lib/php5/sessionclean /var/lib/php5 $(/usr/lib/php5/maxlifetime))
Mar 23 17:15:01 node1 CRON[17036]: (root) CMD (command -v debian-sa1 > /dev/null && debian-sa1 1 1)
The text was updated successfully, but these errors were encountered: