bios: Don't keep sending BIO_FLUSH after first ENOTSUPP.

When a storage device reports that it does not support cache flush, the
GEOM disk layer by default returns ENOTSUPP in response to a BIO_FLUSH
command.

On AWS, local volumes do not advertise themselves as having write-cache
enabled.  When they are selected for L3 on all HDD nodes, the L3
subsystem may inadvertently kick these L3 devices if a BIO_FLUSH command
fails with an ENOTSUPP return code.  The fix is to make GEOM disk return
success (0) when this condition occurs and add a sysctl to make this
error handling config-driven

Reviewed by: imp
Pull Request: #710

sys/geom/geom_disk.c

@@ -71,6 +71,7 @@ struct g_disk_softc {
 	char			led[64];
 	uint32_t		state;
 	struct mtx		 done_mtx;
+	bool                    flush_notsup_succeed;
 };
 
 static g_access_t g_disk_access;
@@ -539,7 +540,7 @@ g_disk_start(struct bio *bp)
 		g_trace(G_T_BIO, "g_disk_flushcache(%s)",
 		    bp->bio_to->name);
 		if (!(dp->d_flags & DISKFLAG_CANFLUSHCACHE)) {
-			error = EOPNOTSUPP;
+			error = (sc->flush_notsup_succeed) ? 0 : EOPNOTSUPP;
 			break;
 		}
 		/*FALLTHROUGH*/
@@ -760,6 +761,10 @@ g_disk_create(void *arg, int flag)
 		    SYSCTL_CHILDREN(sc->sysctl_tree), OID_AUTO, "flags",
 		    CTLTYPE_STRING | CTLFLAG_RD | CTLFLAG_MPSAFE, dp, 0,
 		    g_disk_sysctl_flags, "A", "Report disk flags");
+		SYSCTL_ADD_BOOL(&sc->sysctl_ctx,
+		    SYSCTL_CHILDREN(sc->sysctl_tree), OID_AUTO, "flush_notsup_succeed",
+		    CTLFLAG_RWTUN, &sc->flush_notsup_succeed, sizeof(sc->flush_notsup_succeed),
+		    "Do not return EOPNOTSUPP if there is no cache to flush");
 	}
 	pp->private = sc;
 	dp->d_geom = gp;