Merge pull request #1 from zfsonlinux/master

Merge from upstream

.github/CONTRIBUTING.md

@@ -34,6 +34,7 @@ Helpful resources
 
   * [ZFS on Linux wiki](https://github.com/zfsonlinux/zfs/wiki)
   * [OpenZFS Documentation](http://open-zfs.org/wiki/Developer_resources)
+  * [Git and GitHub for beginners](https://github.com/zfsonlinux/zfs/wiki/Git-and-GitHub-for-beginners)
 
 ## What should I know before I get started?
 

.github/ISSUE_TEMPLATE.md

@@ -1,3 +1,5 @@
+<!-- Please fill out the following template, which will help other contributors address your issue. -->
+
 <!--
 Thank you for reporting an issue.
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,17 +1,19 @@
+<!--- Please fill out the following template, which will help other contributors review your Pull Request. -->
+
 <!--- Provide a general summary of your changes in the Title above -->
 
 <!---
 Documentation on ZFS Buildbot options can be found at
 https://github.com/zfsonlinux/zfs/wiki/Buildbot-Options
 -->
 
-### Description
-<!--- Describe your changes in detail -->
-
 ### Motivation and Context
 <!--- Why is this change required? What problem does it solve? -->
 <!--- If it fixes an open issue, please link to the issue here. -->
 
+### Description
+<!--- Describe your changes in detail -->
+
 ### How Has This Been Tested?
 <!--- Please describe in detail how you tested your changes. -->
 <!--- Include details of your testing environment, and the tests you ran to -->
@@ -36,4 +38,3 @@ https://github.com/zfsonlinux/zfs/wiki/Buildbot-Options
 - [ ] I have added [tests](https://github.com/zfsonlinux/zfs/tree/master/tests) to cover my changes.
 - [ ] All new and existing tests passed.
 - [ ] All commit messages are properly formatted and contain [`Signed-off-by`](https://github.com/zfsonlinux/zfs/blob/master/.github/CONTRIBUTING.md#signed-off-by).
-- [ ] Change has been approved by a ZFS on Linux member.

META

@@ -1,10 +1,10 @@
 Meta:          1
 Name:          zfs
 Branch:        1.0
-Version:       0.7.0
-Release:       1
+Version:       0.8.0
+Release:       rc2
 Release-Tags:  relext
 License:       CDDL
 Author:        OpenZFS on Linux
-Linux-Maximum: 4.17
+Linux-Maximum: 4.19
 Linux-Minimum: 2.6.32

Makefile.am

@@ -21,10 +21,16 @@ endif
 AUTOMAKE_OPTIONS = foreign
 EXTRA_DIST  = autogen.sh copy-builtin
 EXTRA_DIST += config/config.awk config/rpm.am config/deb.am config/tgz.am
-EXTRA_DIST += META AUTHORS COPYRIGHT LICENSE NOTICE README.md
+EXTRA_DIST += META AUTHORS COPYRIGHT LICENSE NEWS NOTICE README.md
 
 @CODE_COVERAGE_RULES@
 
+.PHONY: gitrev
+gitrev:
+	-${top_srcdir}/scripts/make_gitrev.sh
+
+BUILT_SOURCES = gitrev
+
 distclean-local::
 	-$(RM) -R autom4te*.cache
 	-find . \( -name SCCS -o -name BitKeeper -o -name .svn -o -name CVS \
@@ -40,7 +46,8 @@ distclean-local::
 all-local:
 	-${top_srcdir}/scripts/zfs-tests.sh -c
 
-dist-hook:
+dist-hook: gitrev
+	cp ${top_srcdir}/include/zfs_gitrev.h $(distdir)/include; \
 	sed -i 's/Release:[[:print:]]*/Release:      $(RELEASE)/' \
 		$(distdir)/META
 
@@ -61,7 +68,9 @@ install-data-hook:
 	ln -fs zfs.release spl.release
 endif
 
-checkstyle: cstyle shellcheck flake8 commitcheck mancheck testscheck
+codecheck: cstyle shellcheck flake8 mancheck testscheck
+
+checkstyle: codecheck commitcheck
 
 commitcheck:
 	@if git rev-parse --git-dir > /dev/null 2>&1; then \
@@ -79,6 +88,8 @@ shellcheck:
 			$$(find ${top_srcdir}/scripts/*.sh -type f) \
 			$$(find ${top_srcdir}/cmd/zed/zed.d/*.sh -type f) \
 			$$(find ${top_srcdir}/cmd/zpool/zpool.d/* -executable); \
+	else \
+		echo "skipping shellcheck because shellcheck is not installed"; \
 	fi
 
 mancheck:
@@ -87,6 +98,8 @@ mancheck:
 			-o -name 'zpool.8' -o -name 'zdb.8' \
 			-o -name 'zgenhostid.8' | \
 			xargs mandoc -Tlint -Werror; \
+	else \
+		echo "skipping mancheck because mandoc is not installed"; \
 	fi
 
 testscheck:
@@ -104,17 +117,23 @@ cppcheck:
 		cppcheck --quiet --force --error-exitcode=2 --inline-suppr \
 			--suppressions-list=.github/suppressions.txt \
 			-UHAVE_SSE2 -UHAVE_AVX512F -UHAVE_UIO_ZEROCOPY \
-			-UHAVE_DNLC ${top_srcdir}; \
+			${top_srcdir}; \
+	else \
+		echo "skipping cppcheck because cppcheck is not installed"; \
 	fi
 
 paxcheck:
 	@if type scanelf > /dev/null 2>&1; then \
 		${top_srcdir}/scripts/paxcheck.sh ${top_srcdir}; \
+	else \
+		echo "skipping paxcheck because scanelf is not installed"; \
 	fi
 
 flake8:
 	@if type flake8 > /dev/null 2>&1; then \
 		flake8 ${top_srcdir}; \
+	else \
+		echo "skipping flake8 because flake8 is not installed"; \
 	fi
 
 ctags:

NEWS

@@ -0,0 +1,3 @@
+Descriptions of all releases can be found on github:
+
+https://github.com/zfsonlinux/zfs/releases

cmd/arc_summary/arc_summary.py

@@ -204,6 +204,10 @@ def get_arc_summary(Kstat):
     arc_size = Kstat["kstat.zfs.misc.arcstats.size"]
     mru_size = Kstat["kstat.zfs.misc.arcstats.mru_size"]
     mfu_size = Kstat["kstat.zfs.misc.arcstats.mfu_size"]
+    meta_limit = Kstat["kstat.zfs.misc.arcstats.arc_meta_limit"]
+    meta_size = Kstat["kstat.zfs.misc.arcstats.arc_meta_used"]
+    dnode_limit = Kstat["kstat.zfs.misc.arcstats.arc_dnode_limit"]
+    dnode_size = Kstat["kstat.zfs.misc.arcstats.dnode_size"]
     target_max_size = Kstat["kstat.zfs.misc.arcstats.c_max"]
     target_min_size = Kstat["kstat.zfs.misc.arcstats.c_min"]
     target_size = Kstat["kstat.zfs.misc.arcstats.c"]
@@ -228,6 +232,22 @@ def get_arc_summary(Kstat):
         'per': fPerc(target_size, target_max_size),
         'num': fBytes(target_size),
     }
+    output['arc_sizing']['meta_limit'] = {
+        'per': fPerc(meta_limit, target_max_size),
+        'num': fBytes(meta_limit),
+    }
+    output['arc_sizing']['meta_size'] = {
+        'per': fPerc(meta_size, meta_limit),
+        'num': fBytes(meta_size),
+    }
+    output['arc_sizing']['dnode_limit'] = {
+        'per': fPerc(dnode_limit, meta_limit),
+        'num': fBytes(dnode_limit),
+    }
+    output['arc_sizing']['dnode_size'] = {
+        'per': fPerc(dnode_size, dnode_limit),
+        'num': fBytes(dnode_size),
+    }
 
     # ARC Hash Breakdown
     output['arc_hash_break'] = {}
@@ -333,6 +353,26 @@ def _arc_summary(Kstat):
         arc['arc_size_break']['frequently_used_cache_size']['num'],
         )
     )
+    sys.stdout.write("\tMetadata Size (Hard Limit):\t%s\t%s\n" % (
+        arc['arc_sizing']['meta_limit']['per'],
+        arc['arc_sizing']['meta_limit']['num'],
+        )
+    )
+    sys.stdout.write("\tMetadata Size:\t\t\t%s\t%s\n" % (
+        arc['arc_sizing']['meta_size']['per'],
+        arc['arc_sizing']['meta_size']['num'],
+        )
+    )
+    sys.stdout.write("\tDnode Size (Hard Limit):\t%s\t%s\n" % (
+        arc['arc_sizing']['dnode_limit']['per'],
+        arc['arc_sizing']['dnode_limit']['num'],
+        )
+    )
+    sys.stdout.write("\tDnode Size:\t\t\t%s\t%s\n" % (
+        arc['arc_sizing']['dnode_size']['per'],
+        arc['arc_sizing']['dnode_size']['num'],
+        )
+    )
 
     sys.stdout.write("\n")
 

cmd/arc_summary/arc_summary3.py

@@ -109,9 +109,16 @@ def draw_graph(kstats_dict):
     arc_perc = f_perc(arc_stats['size'], arc_stats['c_max'])
     mfu_size = f_bytes(arc_stats['mfu_size'])
     mru_size = f_bytes(arc_stats['mru_size'])
-
-    info_form = 'ARC: {0} ({1})  MFU: {2}  MRU: {3}'
-    info_line = info_form.format(arc_size, arc_perc, mfu_size, mru_size)
+    meta_limit = f_bytes(arc_stats['arc_meta_limit'])
+    meta_size = f_bytes(arc_stats['arc_meta_used'])
+    dnode_limit = f_bytes(arc_stats['arc_dnode_limit'])
+    dnode_size = f_bytes(arc_stats['dnode_size'])
+
+    info_form = ('ARC: {0} ({1})  MFU: {2}  MRU: {3}  META: {4} ({5}) '
+                 'DNODE {6} ({7})')
+    info_line = info_form.format(arc_size, arc_perc, mfu_size, mru_size,
+                                 meta_size, meta_limit, dnode_size,
+                                 dnode_limit)
     info_spc = ' '*int((GRAPH_WIDTH-len(info_line))/2)
     info_line = GRAPH_INDENT+info_spc+info_line
 
@@ -483,6 +490,10 @@ def section_arc(kstats_dict):
     arc_min = arc_stats['c_min']
     mfu_size = arc_stats['mfu_size']
     mru_size = arc_stats['mru_size']
+    meta_limit = arc_stats['arc_meta_limit']
+    meta_size = arc_stats['arc_meta_used']
+    dnode_limit = arc_stats['arc_dnode_limit']
+    dnode_size = arc_stats['dnode_size']
     target_size_ratio = '{0}:1'.format(int(arc_max) // int(arc_min))
 
     prt_2('ARC size (current):',
@@ -498,6 +509,14 @@ def section_arc(kstats_dict):
            f_perc(mfu_size, caches_size), f_bytes(mfu_size))
     prt_i2('Most Recently Used (MRU) cache size:',
            f_perc(mru_size, caches_size), f_bytes(mru_size))
+    prt_i2('Metadata cache size (hard limit):',
+           f_perc(meta_limit, arc_max), f_bytes(meta_limit))
+    prt_i2('Metadata cache size (current):',
+           f_perc(meta_size, meta_limit), f_bytes(meta_size))
+    prt_i2('Dnode cache size (hard limit):',
+           f_perc(dnode_limit, meta_limit), f_bytes(dnode_limit))
+    prt_i2('Dnode cache size (current):',
+           f_perc(dnode_size, dnode_limit), f_bytes(dnode_size))
     print()
 
     print('ARC hash breakdown:')

cmd/arcstat/arcstat.py

@@ -71,7 +71,7 @@
     "pm%":        [3, 100, "Prefetch miss percentage"],
     "mhit":       [4, 1000, "Metadata hits per second"],
     "mmis":       [4, 1000, "Metadata misses per second"],
-    "mread":      [4, 1000, "Metadata accesses per second"],
+    "mread":      [5, 1000, "Metadata accesses per second"],
     "mh%":        [3, 100, "Metadata hit percentage"],
     "mm%":        [3, 100, "Metadata miss percentage"],
     "arcsz":      [5, 1024, "ARC Size"],
@@ -92,6 +92,9 @@
     "l2asize":    [7, 1024, "Actual (compressed) size of the L2ARC"],
     "l2size":     [6, 1024, "Size of the L2ARC"],
     "l2bytes":    [7, 1024, "bytes read per second from the L2ARC"],
+    "grow":       [4, 1000, "ARC Grow disabled"],
+    "need":       [4, 1024, "ARC Reclaim need"],
+    "free":       [4, 1024, "ARC Free memory"],
 }
 
 v = {}
@@ -112,7 +115,6 @@
 d = {}
 out = None
 kstat = None
-float_pobj = re.compile("^[0-9]+(\.[0-9]+)?$")
 
 
 def detailed_usage():
@@ -285,7 +287,7 @@ def init():
             ]
         )
     except getopt.error as msg:
-        sys.stderr.write(msg)
+        sys.stderr.write("Error: %s\n" % str(msg))
         usage()
         opts = None
 
@@ -424,6 +426,10 @@ def calculate():
         v["l2size"] = cur["l2_size"]
         v["l2bytes"] = d["l2_read_bytes"] / sint
 
+    v["grow"] = 0 if cur["arc_no_grow"] else 1
+    v["need"] = cur["arc_need_free"]
+    v["free"] = cur["arc_sys_free"]
+
 
 def main():
     global sint

cmd/mount_zfs/mount_zfs.c

@@ -31,6 +31,7 @@
 #include <sys/mntent.h>
 #include <sys/stat.h>
 #include <libzfs.h>
+#include <libzutil.h>
 #include <locale.h>
 #include <getopt.h>
 #include <fcntl.h>