ZSTD replication compatibility improvements

If the dataset being sent has the ZSTD feature activated, and
the replication compression flag is specified, we set the ZSTD
replication feature flag, so the receiving side will reject the
stream if it does not support ZSTD.

Only send ZSTD compressed embedded block pointers if the ZSTD
replication flag is already set.

`zfs send -c` will send ZSTD compressed blocks
`zfs send -e` will send embedded LZ4 block pointers but convert
    ZSTD eBP's into regular blocks
`zfs send -ec` will send ZSTD compressed blocks and eBPs

Signed-off-by: Allan Jude <allan@klarasystems.com>

Author: allanjude

module/zfs/dmu_send.c

@@ -864,6 +864,14 @@ send_do_embed(const blkptr_t *bp, uint64_t featureflags)
 	    !(featureflags & DMU_BACKUP_FEATURE_LZ4)))
 		return (B_FALSE);
 
+	/*
+	 * If we have not set the ZSTD feature flag, we can't send ZSTD
+	 * compressed embedded blocks, as the receiver may not support them.
+	 */
+	if ((BP_GET_COMPRESS(bp) == ZIO_COMPRESS_ZSTD &&
+	    !(featureflags & DMU_BACKUP_FEATURE_ZSTD)))
+		return (B_FALSE);
+
 	/*
 	 * Embed type must be explicitly enabled.
 	 */
@@ -1967,9 +1975,13 @@ setup_featureflags(struct dmu_send_params *dspp, objset_t *os,
 		*featureflags |= DMU_BACKUP_FEATURE_LZ4;
 	}
 
+	/*
+	 * We specifically do not include DMU_BACKUP_FEATURE_EMBED_DATA here to
+	 * allow sending ZSTD compressed datasets to a receiver that does not
+	 * support ZSTD
+	 */
 	if ((*featureflags &
-	    (DMU_BACKUP_FEATURE_EMBED_DATA | DMU_BACKUP_FEATURE_COMPRESSED |
-	    DMU_BACKUP_FEATURE_RAW)) != 0 &&
+	    (DMU_BACKUP_FEATURE_COMPRESSED | DMU_BACKUP_FEATURE_RAW)) != 0 &&
 	    dsl_dataset_feature_is_active(to_ds, SPA_FEATURE_ZSTD_COMPRESS)) {
 		*featureflags |= DMU_BACKUP_FEATURE_ZSTD;
 	}