Add "features" property for zpool feature sets [2021]

Property to allow sets of features to be specified;
influences the behavior of 'zpool upgrade' and 'zpool create'.
Initial man page changes included.

(This is a clean rebase against master as of 2021-01-15.)

Brief synopsis:

zpool create -o features=all|none|file[,file...] pool vdev...

features = all : request that all features be enabled
features = none : request that no features be enabled
features = file[,file...] : read features from the specified files.
Only features present in *all* files will be enabled on the resulting
pool.

Only affects zpool create, zpool upgrade and zpool status.

ABI changes in libzfs:

* Added function "zpool_load_features" to load and parse feature files.
* Added ZPOOL_PROP_FEATURES to the pool properties enum
* Added ZPOOL_STATUS_FEAT_ERR to the pool status enum

Signed-off-by: Colm Buckley <colm@tuatha.org>

cmd/zpool/zpool_main.c

@@ -31,6 +31,7 @@
  * Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
  * Copyright (c) 2017, Intel Corporation.
  * Copyright (c) 2019, loli10K <ezomori.nozomu@gmail.com>
+ * Portions Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
  */
 
 #include <assert.h>
@@ -124,6 +125,9 @@ static int zpool_do_version(int, char **);
 
 static int zpool_do_wait(int, char **);
 
+static zpool_features_status_t zpool_do_load_features(
+    const char *, boolean_t *);
+
 /*
  * These libumem hooks provide a reasonable set of defaults for the allocator's
  * debugging facilities.
@@ -782,6 +786,7 @@ add_prop_list(const char *propname, char *propval, nvlist_t **props,
 
 	if (poolprop) {
 		const char *vname = zpool_prop_to_name(ZPOOL_PROP_VERSION);
+		const char *fname = zpool_prop_to_name(ZPOOL_PROP_FEATURES);
 
 		if ((prop = zpool_name_to_prop(propname)) == ZPOOL_PROP_INVAL &&
 		    !zpool_prop_feature(propname)) {
@@ -804,6 +809,19 @@ add_prop_list(const char *propname, char *propval, nvlist_t **props,
 			return (2);
 		}
 
+		/*
+		 * features property and version should not be specified
+		 * at the same time.
+		 */
+		if ((prop == ZPOOL_PROP_FEATURES &&
+		    nvlist_exists(proplist, vname)) ||
+		    (prop == ZPOOL_PROP_VERSION &&
+		    nvlist_exists(proplist, fname))) {
+			(void) fprintf(stderr, gettext("'features' and "
+			    "'version' properties cannot be specified "
+			    "together\n"));
+			return (2);
+		}
 
 		if (zpool_prop_feature(propname))
 			normnm = propname;
@@ -1374,7 +1392,11 @@ zpool_do_create(int argc, char **argv)
 {
 	boolean_t force = B_FALSE;
 	boolean_t dryrun = B_FALSE;
-	boolean_t enable_all_pool_feat = B_TRUE;
+
+	boolean_t auto_enable_pool_feat = B_TRUE;
+	boolean_t pool_features[SPA_FEATURES];
+	boolean_t features_set = B_FALSE;
+
 	int c;
 	nvlist_t *nvroot = NULL;
 	char *poolname;
@@ -1396,7 +1418,7 @@ zpool_do_create(int argc, char **argv)
 			dryrun = B_TRUE;
 			break;
 		case 'd':
-			enable_all_pool_feat = B_FALSE;
+			auto_enable_pool_feat = B_FALSE;
 			break;
 		case 'R':
 			altroot = optarg;
@@ -1434,9 +1456,15 @@ zpool_do_create(int argc, char **argv)
 				ver = strtoull(propval, &end, 10);
 				if (*end == '\0' &&
 				    ver < SPA_VERSION_FEATURES) {
-					enable_all_pool_feat = B_FALSE;
+					auto_enable_pool_feat = B_FALSE;
 				}
 			}
+			if (zpool_name_to_prop(optarg) == ZPOOL_PROP_FEATURES) {
+				features_set = B_TRUE;
+				if (zpool_do_load_features(propval,
+				    pool_features) != ZPOOL_FEATURES_OK)
+					goto errout;
+			}
 			if (zpool_name_to_prop(optarg) == ZPOOL_PROP_ALTROOT)
 				altroot = propval;
 			break;
@@ -1647,14 +1675,25 @@ zpool_do_create(int argc, char **argv)
 			 * Only features contained in props will be enabled:
 			 * remove from the nvlist every ZFS_FEATURE_DISABLED
 			 * value and add every missing ZFS_FEATURE_ENABLED if
-			 * enable_all_pool_feat is set.
+			 * auto_enable_pool_feat is set. If 'features' property
+			 * is set, only add features from that list.
 			 */
 			if (!nvlist_lookup_string(props, propname, &propval)) {
 				if (strcmp(propval, ZFS_FEATURE_DISABLED) == 0)
 					(void) nvlist_remove_all(props,
 					    propname);
-			} else if (enable_all_pool_feat &&
-			    feat->fi_zfs_mod_supported) {
+			} else if (
+			    auto_enable_pool_feat &&
+			    feat->fi_zfs_mod_supported &&
+			    (!features_set || pool_features[i])) {
+				/*
+				 * How this works:
+				 * If 'auto_enable_pool_feat' is set,
+				 * we try to enable "all" supportedfeatures.
+				 * But if we have "-o features", then
+				 * only enable features from the set
+				 * we read into pool_features.
+				 */
 				ret = add_prop_list(propname,
 				    ZFS_FEATURE_ENABLED, &props, B_TRUE);
 				if (ret != 0)
@@ -2674,8 +2713,14 @@ show_import(nvlist_t *config)
 
 	case ZPOOL_STATUS_FEAT_DISABLED:
 		printf_color(ANSI_BOLD, gettext("status: "));
-		printf_color(ANSI_YELLOW, gettext("Some supported features are "
-		    "not enabled on the pool.\n"));
+		printf_color(ANSI_YELLOW, gettext("Some supported and "
+		    "requested features are not enabled on the pool.\n"));
+		break;
+
+	case ZPOOL_STATUS_FEAT_ERR:
+		printf_color(ANSI_BOLD, gettext("status: "));
+		printf_color(ANSI_YELLOW, gettext("Error reading or parsing "
+		    "the file indicated by the 'features' property.\n"));
 		break;
 
 	case ZPOOL_STATUS_UNSUP_FEAT_READ:
@@ -2767,6 +2812,11 @@ show_import(nvlist_t *config)
 			    "imported using its name or numeric identifier, "
 			    "though\n\tsome features will not be available "
 			    "without an explicit 'zpool upgrade'.\n"));
+		} else if (reason == ZPOOL_STATUS_FEAT_ERR) {
+			(void) printf(gettext(" action: The pool can be "
+			    "imported using its name or numeric identifier, "
+			    "though the file indicated by its 'features' "
+			    "property cannot be parsed at this time.\n"));
 		} else if (reason == ZPOOL_STATUS_HOSTID_MISMATCH) {
 			(void) printf(gettext(" action: The pool can be "
 			    "imported using its name or numeric "
@@ -7942,7 +7992,8 @@ status_callback(zpool_handle_t *zhp, void *data)
 	if (cbp->cb_explain &&
 	    (reason == ZPOOL_STATUS_OK ||
 	    reason == ZPOOL_STATUS_VERSION_OLDER ||
-	    reason == ZPOOL_STATUS_FEAT_DISABLED)) {
+	    reason == ZPOOL_STATUS_FEAT_DISABLED ||
+	    reason == ZPOOL_STATUS_FEAT_ERR)) {
 		if (!cbp->cb_allpools) {
 			(void) printf(gettext("pool '%s' is healthy\n"),
 			    zpool_get_name(zhp));
@@ -8127,6 +8178,18 @@ status_callback(zpool_handle_t *zhp, void *data)
 		    "the features. See zpool-features(5) for details.\n"));
 		break;
 
+	case ZPOOL_STATUS_FEAT_ERR:
+		printf_color(ANSI_BOLD, gettext("status: "));
+		printf_color(ANSI_YELLOW, gettext("This pool has a feature "
+		    "list specified, but it could not be read/parsed at this "
+		    "time. The pool can still be used, but this should be "
+		    "investigated.\n"));
+		printf_color(ANSI_BOLD, gettext("action: "));
+		printf_color(ANSI_YELLOW, gettext("Check the value of the "
+		    "'features' property against the "
+		    "appropriate file in " ZPOOL_FEATURES_DIR ".\n"));
+		break;
+
 	case ZPOOL_STATUS_UNSUP_FEAT_READ:
 		printf_color(ANSI_BOLD, gettext("status: "));
 		printf_color(ANSI_YELLOW, gettext("The pool cannot be accessed "
@@ -8625,11 +8688,21 @@ upgrade_enable_all(zpool_handle_t *zhp, int *countp)
 	boolean_t firstff = B_TRUE;
 	nvlist_t *enabled = zpool_get_features(zhp);
 
+	char features[ZFS_MAXPROPLEN];
+	if (zpool_get_prop(zhp, ZPOOL_PROP_FEATURES, features,
+	    ZFS_MAXPROPLEN, NULL, B_FALSE) != 0)
+		features[0] = '\0';
+
+	boolean_t pool_features[SPA_FEATURES];
+	if (zpool_do_load_features(features, pool_features) !=
+	    ZPOOL_FEATURES_OK)
+		return (-1);
+
 	count = 0;
 	for (i = 0; i < SPA_FEATURES; i++) {
 		const char *fname = spa_feature_table[i].fi_uname;
 		const char *fguid = spa_feature_table[i].fi_guid;
-		if (!nvlist_exists(enabled, fguid)) {
+		if (!nvlist_exists(enabled, fguid) && pool_features[i]) {
 			char *propname;
 			verify(-1 != asprintf(&propname, "feature@%s", fname));
 			ret = zpool_set_prop(zhp, propname,
@@ -8847,7 +8920,7 @@ upgrade_one(zpool_handle_t *zhp, void *data)
 			printnl = B_TRUE;
 		} else if (cur_version == SPA_VERSION) {
 			(void) printf(gettext("Pool '%s' already has all "
-			    "supported features enabled.\n"),
+			    "supported and requested features enabled.\n"),
 			    zpool_get_name(zhp));
 		}
 	}
@@ -9008,8 +9081,8 @@ zpool_do_upgrade(int argc, char **argv)
 				(void) printf(gettext("All pools are already "
 				    "formatted using feature flags.\n\n"));
 				(void) printf(gettext("Every feature flags "
-				    "pool already has all supported features "
-				    "enabled.\n"));
+				    "pool already has all supported and "
+				    "requested features enabled.\n"));
 			} else {
 				(void) printf(gettext("All pools are already "
 				    "formatted with version %llu or higher.\n"),
@@ -9035,7 +9108,7 @@ zpool_do_upgrade(int argc, char **argv)
 
 		if (cb.cb_first) {
 			(void) printf(gettext("Every feature flags pool has "
-			    "all supported features enabled.\n"));
+			    "all supported and requested features enabled.\n"));
 		} else {
 			(void) printf(gettext("\n"));
 		}
@@ -10339,6 +10412,38 @@ zpool_do_version(int argc, char **argv)
 	return (0);
 }
 
+/*
+ * Do zpool_load_features() and print error message on failure
+ */
+static zpool_features_status_t
+zpool_do_load_features(const char *feat, boolean_t *list)
+{
+	char badword[ZFS_MAXPROPLEN];
+	char badfile[MAXPATHLEN];
+	zpool_features_status_t ret;
+
+	switch (ret = zpool_load_features(feat, list, badword, badfile)) {
+	case ZPOOL_FEATURES_OK:
+		break;
+	case ZPOOL_FEATURES_READERR:
+		(void) fprintf(stderr, gettext("error reading feature file "
+		    "'%s'\n"), badfile);
+		break;
+	case ZPOOL_FEATURES_BADFILE:
+		(void) fprintf(stderr, gettext("feature file '%s' not newline-"
+		    "terminated\n"), badfile);
+		break;
+	case ZPOOL_FEATURES_BADWORD:
+		(void) fprintf(stderr, gettext("unknown feature '%s' in "
+		    "feature file '%s'\n"), badword, badfile);
+		break;
+	case ZPOOL_FEATURES_NOFILES:
+		(void) fprintf(stderr, gettext("no feature files specified\n"));
+		break;
+	}
+	return (ret);
+}
+
 int
 main(int argc, char **argv)
 {

include/libzfs.h

@@ -28,6 +28,7 @@
  * Copyright 2016 Nexenta Systems, Inc.
  * Copyright (c) 2017 Open-E, Inc. All Rights Reserved.
  * Copyright (c) 2019 Datto Inc.
+ * Portions Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
  */
 
 #ifndef	_LIBZFS_H
@@ -391,6 +392,7 @@ typedef enum {
 	ZPOOL_STATUS_REBUILDING,	/* device being rebuilt */
 	ZPOOL_STATUS_REBUILD_SCRUB,	/* recommend scrubbing the pool */
 	ZPOOL_STATUS_NON_NATIVE_ASHIFT,	/* (e.g. 512e dev with ashift of 9) */
+	ZPOOL_STATUS_FEAT_ERR,		/* can't parse 'features' property */
 
 	/*
 	 * Finally, the following indicates a healthy pool.
@@ -912,6 +914,20 @@ int zfs_smb_acl_rename(libzfs_handle_t *, char *, char *, char *, char *);
 extern int zpool_enable_datasets(zpool_handle_t *, const char *, int);
 extern int zpool_disable_datasets(zpool_handle_t *, boolean_t);
 
+/*
+ * Parse a features file for -o features
+ */
+typedef enum {
+	ZPOOL_FEATURES_OK = 0,
+	ZPOOL_FEATURES_READERR = -1,
+	ZPOOL_FEATURES_BADFILE = -2,
+	ZPOOL_FEATURES_BADWORD = -3,
+	ZPOOL_FEATURES_NOFILES = -4
+} zpool_features_status_t;
+
+extern zpool_features_status_t zpool_load_features(const char *,
+    boolean_t *, char *, char *);
+
 #ifdef __FreeBSD__
 
 /*

include/sys/fs/zfs.h

@@ -27,10 +27,10 @@
  * Copyright (c) 2014 Integros [integros.com]
  * Copyright (c) 2017, Intel Corporation.
  * Copyright (c) 2019 Datto Inc.
+ * Portions Copyright 2010 Robert Milkowski
+ * Portions Copyright (c) 2021, Colm Buckley <colm@tuatha.org>
  */
 
-/* Portions Copyright 2010 Robert Milkowski */
-
 #ifndef	_SYS_FS_ZFS_H
 #define	_SYS_FS_ZFS_H
 
@@ -246,6 +246,7 @@ typedef enum {
 	ZPOOL_PROP_CHECKPOINT,
 	ZPOOL_PROP_LOAD_GUID,
 	ZPOOL_PROP_AUTOTRIM,
+	ZPOOL_PROP_FEATURES,
 	ZPOOL_NUM_PROPS
 } zpool_prop_t;
 
@@ -733,6 +734,7 @@ typedef struct zpool_load_policy {
 #define	ZPOOL_CONFIG_ALLOCATION_BIAS	"alloc_bias"	/* not stored on disk */
 #define	ZPOOL_CONFIG_EXPANSION_TIME	"expansion_time"	/* not stored */
 #define	ZPOOL_CONFIG_REBUILD_STATS	"org.openzfs:rebuild_stats"
+#define	ZPOOL_CONFIG_FEATURES		"features"
 
 /*
  * The persistent vdev state is stored as separate values rather than a single
@@ -845,6 +847,10 @@ typedef struct zpool_load_policy {
  */
 #define	ZPOOL_CACHE_BOOT	"/boot/zfs/zpool.cache"
 #define	ZPOOL_CACHE		"/etc/zfs/zpool.cache"
+/*
+ * Default directory for zpool features files
+ */
+#define	ZPOOL_FEATURES_DIR	"/etc/zfs/features.d"
 /*
  * vdev states are ordered from least to most healthy.
  * A vdev that's CANT_OPEN or below is considered unusable.

include/sys/spa_impl.h

@@ -424,6 +424,8 @@ struct spa {
 	int		spa_waiters;		/* number of waiting threads */
 	boolean_t	spa_waiters_cancel;	/* waiters should return */
 
+	char		*spa_features;		/* desired feature file(s) */
+
 	/*
 	 * spa_refcount & spa_config_lock must be the last elements
 	 * because zfs_refcount_t changes size based on compilation options.