root@node4:~# arc_summary ------------------------------------------------------------------------ ZFS Subsystem Report Tue Nov 21 17:11:09 2017 ARC Summary: (HEALTHY) Memory Throttle Count: 0 ARC Misc: Deleted: 608.59m Mutex Misses: 50.43m Evict Skips: 50.43m ARC Size: 100.47% 1.00 GiB Target Size: (Adaptive) 100.00% 1.00 GiB Min Size (Hard Limit): 3.12% 32.00 MiB Max Size (High Water): 32:1 1.00 GiB ARC Size Breakdown: Recently Used Cache Size: 90.57% 931.79 MiB Frequently Used Cache Size: 9.43% 96.98 MiB ARC Hash Breakdown: Elements Max: 293.59k Elements Current: 67.68% 198.71k Collisions: 13.78m Chain Max: 4 Chains: 2.10k ARC Total accesses: 887.77m Cache Hit Ratio: 29.78% 264.41m Cache Miss Ratio: 70.22% 623.36m Actual Hit Ratio: 27.31% 242.45m Data Demand Efficiency: 33.31% 656.36m Data Prefetch Efficiency: 8.40% 169.60m CACHE HITS BY CACHE LIST: Most Recently Used: 89.70% 237.19m Most Frequently Used: 1.99% 5.27m Most Recently Used Ghost: 6.00% 15.86m Most Frequently Used Ghost: 2.49% 6.58m CACHE HITS BY DATA TYPE: Demand Data: 82.68% 218.62m Prefetch Data: 5.39% 14.25m Demand Metadata: 8.88% 23.47m Prefetch Metadata: 3.05% 8.07m CACHE MISSES BY DATA TYPE: Demand Data: 70.22% 437.74m Prefetch Data: 24.92% 155.34m Demand Metadata: 3.95% 24.63m Prefetch Metadata: 0.90% 5.64m DMU Prefetch Efficiency: 1.36b Hit Ratio: 3.08% 41.97m Miss Ratio: 96.92% 1.32b ZFS Tunable: zvol_volmode 1 l2arc_headroom 2 dbuf_cache_max_shift 5 zfs_free_leak_on_eio 0 zfs_free_max_blocks 100000 zfs_read_chunk_size 1048576 metaslab_preload_enabled 1 zfs_dedup_prefetch 0 zfs_txg_history 0 zfs_scrub_delay 4 zfs_vdev_async_read_max_active 3 zfs_read_history 0 zfs_arc_sys_free 0 l2arc_write_max 8388608 zil_slog_bulk 786432 zfs_dbuf_state_index 0 zfs_sync_taskq_batch_pct 75 metaslab_debug_unload 0 zvol_inhibit_dev 0 zfs_abd_scatter_enabled 1 zfs_arc_pc_percent 0 zfetch_max_streams 8 zfs_recover 0 metaslab_fragmentation_factor_enabled 1 zfs_deadman_checktime_ms 5000 zfs_sync_pass_rewrite 2 zfs_object_mutex_size 64 zfs_arc_min_prefetch_lifespan 0 zfs_arc_meta_prune 10000 zfs_read_history_hits 0 zfetch_max_distance 8388608 l2arc_norw 0 zfs_dirty_data_max_percent 10 zfs_per_txg_dirty_frees_percent 30 zfs_arc_meta_min 0 metaslabs_per_vdev 200 zfs_arc_meta_adjust_restarts 4096 spa_load_verify_maxinflight 10000 spa_load_verify_metadata 1 zfs_multihost_history 0 zfs_send_corrupt_data 0 zfs_delay_min_dirty_percent 60 zfs_vdev_sync_read_max_active 10 zfs_dbgmsg_enable 0 zfs_metaslab_segment_weight_enabled 1 zio_requeue_io_start_cut_in_line 1 l2arc_headroom_boost 200 zfs_zevent_cols 80 zfs_dmu_offset_next_sync 0 spa_config_path /etc/zfs/zpool.cache zfs_vdev_cache_size 0 dbuf_cache_hiwater_pct 10 zfs_multihost_interval 1000 zfs_multihost_fail_intervals 5 zio_dva_throttle_enabled 1 zfs_vdev_sync_write_min_active 10 zfs_vdev_scrub_max_active 2 ignore_hole_birth 1 zvol_major 230 zil_replay_disable 0 zfs_dirty_data_max_max_percent 25 zfs_expire_snapshot 300 zfs_sync_pass_deferred_free 2 spa_asize_inflation 24 dmu_object_alloc_chunk_shift 7 zfs_vdev_mirror_rotating_seek_offset 1048576 l2arc_feed_secs 1 zfs_autoimport_disable 1 zfs_arc_p_aggressive_disable 1 zfs_zevent_len_max 256 zfs_arc_meta_limit_percent 75 l2arc_noprefetch 1 zfs_vdev_raidz_impl [fastest] original scalar sse2 ssse3 zfs_arc_meta_limit 0 zfs_flags 0 zfs_dirty_data_max_max 4294967296 zfs_arc_average_blocksize 8192 zfs_vdev_cache_bshift 16 zfs_vdev_async_read_min_active 1 zfs_arc_dnode_reduce_percent 10 zfs_free_bpobj_enabled 1 zfs_arc_grow_retry 0 zfs_vdev_mirror_rotating_inc 0 l2arc_feed_again 1 zfs_vdev_mirror_non_rotating_inc 0 zfs_arc_lotsfree_percent 10 zfs_zevent_console 0 zvol_prefetch_bytes 131072 zfs_free_min_time_ms 1000 zfs_arc_dnode_limit_percent 10 zio_taskq_batch_pct 75 dbuf_cache_max_bytes 33554432 spa_load_verify_data 1 zfs_delete_blocks 20480 zfs_vdev_mirror_non_rotating_seek_inc 1 zfs_multihost_import_intervals 10 zfs_dirty_data_max 4294967296 zfs_vdev_async_write_max_active 10 zfs_dbgmsg_maxsize 4194304 zfs_nocacheflush 0 zfetch_array_rd_sz 1048576 zfs_arc_meta_strategy 1 zfs_dirty_data_sync 67108864 zvol_max_discard_blocks 16384 zvol_threads 32 zfs_vdev_async_write_active_max_dirty_percent 60 zfs_arc_p_dampener_disable 1 zfs_txg_timeout 5 metaslab_aliquot 524288 zfs_mdcomp_disable 0 zfs_vdev_sync_read_min_active 10 zfs_arc_dnode_limit 0 dbuf_cache_lowater_pct 10 zfs_abd_scatter_max_order 10 metaslab_debug_load 0 zfs_vdev_aggregation_limit 131072 metaslab_lba_weighting_enabled 1 zfs_vdev_scheduler noop zfs_vdev_scrub_min_active 1 zfs_no_scrub_io 0 zfs_vdev_cache_max 16384 zfs_scan_idle 50 zfs_arc_shrink_shift 0 spa_slop_shift 5 zfs_vdev_mirror_rotating_seek_inc 5 zfs_deadman_synctime_ms 1000000 send_holes_without_birth_time 1 metaslab_bias_enabled 1 zvol_request_sync 0 zfs_admin_snapshot 1 zfs_no_scrub_prefetch 0 zfs_metaslab_fragmentation_threshold 70 zfs_max_recordsize 1048576 zfs_arc_min 33554432 zfs_nopwrite_enabled 1 zfs_arc_p_min_shift 0 zfs_multilist_num_sublists 0 zfs_vdev_queue_depth_pct 1000 zfs_mg_fragmentation_threshold 85 l2arc_write_boost 8388608 zfs_prefetch_disable 0 l2arc_feed_min_ms 200 zio_delay_max 30000 zfs_vdev_write_gap_limit 4096 zfs_pd_bytes_max 52428800 zfs_scan_min_time_ms 1000 zfs_resilver_min_time_ms 3000 zfs_delay_scale 500000 zfs_vdev_async_write_active_min_dirty_percent 30 zfs_vdev_sync_write_max_active 10 zfs_mg_noalloc_threshold 0 zfs_deadman_enabled 1 zfs_resilver_delay 2 zfs_metaslab_switch_threshold 2 zfs_arc_max 1073741824 zfs_top_maxinflight 32 zfetch_min_sec_reap 2 zfs_immediate_write_sz 32768 zfs_vdev_async_write_min_active 2 zfs_sync_pass_dont_compress 5 zfs_vdev_read_gap_limit 32768 zfs_compressed_arc_enabled 1 zfs_vdev_max_active 1000 --- root@node4:~# zpool status -v pool: dpool state: ONLINE scan: scrub repaired 0B in 2h56m with 0 errors on Sun Nov 12 03:20:04 2017 config: NAME STATE READ WRITE CKSUM dpool ONLINE 0 0 0 mirror-0 ONLINE 0 0 0 sdc1 ONLINE 0 0 0 sdd1 ONLINE 0 0 0 errors: No known data errors pool: rpool state: ONLINE scan: scrub repaired 0B in 0h35m with 0 errors on Sun Nov 12 00:59:47 2017 config: NAME STATE READ WRITE CKSUM rpool ONLINE 0 0 0 mirror-0 ONLINE 0 0 0 sda2 ONLINE 0 0 0 sdb2 ONLINE 0 0 0 errors: No known data errors --- root@node4:~# zpool list NAME SIZE ALLOC FREE EXPANDSZ FRAG CAP DEDUP HEALTH ALTROOT dpool 928G 282G 646G - 18% 30% 1.00x ONLINE - rpool 238G 86.3G 152G - 69% 36% 1.00x ONLINE - --- root@node4:~# zpool get all rpool NAME PROPERTY VALUE SOURCE rpool size 238G - rpool capacity 36% - rpool altroot - default rpool health ONLINE - rpool guid 1793528944452485888 - rpool version - default rpool bootfs rpool/ROOT/pve-1 local rpool delegation on default rpool autoreplace off default rpool cachefile - default rpool failmode wait default rpool listsnapshots off default rpool autoexpand off default rpool dedupditto 0 default rpool dedupratio 1.00x - rpool free 152G - rpool allocated 86.3G - rpool readonly off - rpool ashift 12 local rpool comment - default rpool expandsize - - rpool freeing 0 - rpool fragmentation 69% - rpool leaked 0 - rpool multihost off default rpool feature@async_destroy enabled local rpool feature@empty_bpobj active local rpool feature@lz4_compress active local rpool feature@multi_vdev_crash_dump enabled local rpool feature@spacemap_histogram active local rpool feature@enabled_txg active local rpool feature@hole_birth active local rpool feature@extensible_dataset active local rpool feature@embedded_data active local rpool feature@bookmarks enabled local rpool feature@filesystem_limits enabled local rpool feature@large_blocks enabled local rpool feature@large_dnode enabled local rpool feature@sha512 enabled local rpool feature@skein enabled local rpool feature@edonr enabled local rpool feature@userobj_accounting active local --- root@node4:~# zpool get all dpool NAME PROPERTY VALUE SOURCE dpool size 928G - dpool capacity 30% - dpool altroot - default dpool health ONLINE - dpool guid 6751175486019907265 - dpool version - default dpool bootfs - default dpool delegation on default dpool autoreplace off default dpool cachefile - default dpool failmode wait default dpool listsnapshots off default dpool autoexpand off default dpool dedupditto 0 default dpool dedupratio 1.00x - dpool free 646G - dpool allocated 282G - dpool readonly off - dpool ashift 12 local dpool comment - default dpool expandsize - - dpool freeing 0 - dpool fragmentation 18% - dpool leaked 0 - dpool multihost off default dpool feature@async_destroy enabled local dpool feature@empty_bpobj active local dpool feature@lz4_compress active local dpool feature@multi_vdev_crash_dump enabled local dpool feature@spacemap_histogram active local dpool feature@enabled_txg active local dpool feature@hole_birth active local dpool feature@extensible_dataset active local dpool feature@embedded_data active local dpool feature@bookmarks enabled local dpool feature@filesystem_limits enabled local dpool feature@large_blocks enabled local dpool feature@large_dnode enabled local dpool feature@sha512 enabled local dpool feature@skein enabled local dpool feature@edonr enabled local dpool feature@userobj_accounting active local --- root@node4:~# zfs get all rpool NAME PROPERTY VALUE SOURCE rpool type filesystem - rpool creation Mon Jul 24 23:25 2017 - rpool used 93.1G - rpool available 107G - rpool referenced 96K - rpool compressratio 1.23x - rpool mounted yes - rpool quota 200G local rpool reservation none default rpool recordsize 128K default rpool mountpoint /rpool default rpool sharenfs off default rpool checksum on default rpool compression on local rpool atime off local rpool devices on default rpool exec on default rpool setuid on default rpool readonly off default rpool zoned off default rpool snapdir hidden default rpool aclinherit restricted default rpool createtxg 1 - rpool canmount on default rpool xattr on default rpool copies 1 default rpool version 5 - rpool utf8only off - rpool normalization none - rpool casesensitivity sensitive - rpool vscan off default rpool nbmand off default rpool sharesmb off default rpool refquota none default rpool refreservation none default rpool guid 1347706894154173560 - rpool primarycache all default rpool secondarycache all default rpool usedbysnapshots 0B - rpool usedbydataset 96K - rpool usedbychildren 93.1G - rpool usedbyrefreservation 0B - rpool logbias latency default rpool dedup off default rpool mlslabel none default rpool sync standard local rpool dnodesize legacy default rpool refcompressratio 1.00x - rpool written 96K - rpool logicalused 105G - rpool logicalreferenced 40K - rpool volmode default default rpool filesystem_limit none default rpool snapshot_limit none default rpool filesystem_count none default rpool snapshot_count none default rpool snapdev hidden default rpool acltype off default rpool context none default rpool fscontext none default rpool defcontext none default rpool rootcontext none default rpool relatime off default rpool redundant_metadata all default rpool overlay off default --- root@node4:~# zfs get all dpool NAME PROPERTY VALUE SOURCE dpool type filesystem - dpool creation Tue Jul 25 0:51 2017 - dpool used 282G - dpool available 617G - dpool referenced 122G - dpool compressratio 1.12x - dpool mounted yes - dpool quota 900G local dpool reservation none default dpool recordsize 128K default dpool mountpoint /dpool default dpool sharenfs off default dpool checksum on default dpool compression on local dpool atime off local dpool devices on default dpool exec on default dpool setuid on default dpool readonly off default dpool zoned off default dpool snapdir hidden default dpool aclinherit restricted default dpool createtxg 1 - dpool canmount on default dpool xattr on default dpool copies 1 default dpool version 5 - dpool utf8only off - dpool normalization none - dpool casesensitivity sensitive - dpool vscan off default dpool nbmand off default dpool sharesmb off default dpool refquota none default dpool refreservation none default dpool guid 12868232080626748881 - dpool primarycache all default dpool secondarycache all default dpool usedbysnapshots 0B - dpool usedbydataset 122G - dpool usedbychildren 160G - dpool usedbyrefreservation 0B - dpool logbias latency default dpool dedup off default dpool mlslabel none default dpool sync standard local dpool dnodesize legacy default dpool refcompressratio 1.02x - dpool written 122G - dpool logicalused 316G - dpool logicalreferenced 125G - dpool volmode default default dpool filesystem_limit none default dpool snapshot_limit none default dpool filesystem_count none default dpool snapshot_count none default dpool snapdev hidden default dpool acltype off default dpool context none default dpool fscontext none default dpool defcontext none default dpool rootcontext none default dpool relatime off local dpool redundant_metadata all default dpool overlay off default ---