Add FreeBSD support to ZoL

- Import FreeBSD specific parts
- Update tests to work on FreeBSD

Signed-off-by: Matt Macy <mmacy@FreeBSD.org>
Co-authored-by: Ryan Moeller <ryan@iXsystems.com>
Signed-off-by: Ryan Moeller <ryan@iXsystems.com>

Requires-builders: amazon2, debian10, freebsd12, freebsd13, style

TEST_ZTEST_SKIP="yes"

.github/suppressions.txt

@@ -1,3 +1,6 @@
 preprocessorErrorDirective:./module/zfs/vdev_raidz_math_avx512f.c:243
 preprocessorErrorDirective:./module/zfs/vdev_raidz_math_sse2.c:266
-
+uninitvar:module/os/freebsd/zfs/vdev_geom.c
+uninitvar:module/os/freebsd/zfs/zfs_vfsops.c
+uninitvar:module/os/freebsd/spl/spl_zone.c
+uninitvar:lib/libzutil/os/freebsd/zutil_import_os.c

.gitignore

@@ -62,4 +62,9 @@ cscope.*
 *.patch
 *.orig
 *.log
+*.tmp
 venv
+
+*.so
+*.so.debug
+*.so.full

Makefile.am

@@ -106,6 +106,7 @@ commitcheck:
 cstyle:
 	@find ${top_srcdir} -name build -prune -o -name '*.[hc]' \
 		! -name 'zfs_config.*' ! -name '*.mod.c' -type f \
+		! -name '*_if*.h' \
 		-exec ${top_srcdir}/scripts/cstyle.pl -cpP {} \+
 
 filter_executable = -exec test -x '{}' \; -print

cmd/Makefile.am

@@ -1,10 +1,10 @@
 SUBDIRS  = zfs zpool zdb zhack zinject zstreamdump ztest
-SUBDIRS += fsck_zfs vdev_id raidz_test zgenhostid
+SUBDIRS += fsck_zfs vdev_id raidz_test
 
 if USING_PYTHON
 SUBDIRS += arcstat arc_summary dbufstat
 endif
 
 if BUILD_LINUX
-SUBDIRS += mount_zfs zed zvol_id zvol_wait
+SUBDIRS += mount_zfs zed zgenhostid zvol_id zvol_wait
 endif

cmd/zpool/Makefile.am

@@ -11,6 +11,10 @@ zpool_SOURCES = \
 	zpool_util.h \
 	zpool_vdev.c
 
+if BUILD_FREEBSD
+zpool_SOURCES += os/freebsd/zpool_vdev_os.c
+endif
+
 if BUILD_LINUX
 zpool_SOURCES += os/linux/zpool_vdev_os.c
 endif
@@ -20,6 +24,9 @@ zpool_LDADD = \
 	$(top_builddir)/lib/libuutil/libuutil.la \
 	$(top_builddir)/lib/libzfs/libzfs.la
 
+if BUILD_FREEBSD
+zpool_LDADD += -L/usr/local/lib -lintl -lgeom
+endif
 zpool_LDADD += -lm $(LIBBLKID)
 
 zpoolconfdir = $(sysconfdir)/zfs/zpool.d

cmd/zpool/os/freebsd/zpool_vdev_os.c

@@ -0,0 +1,136 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2018 by Delphix. All rights reserved.
+ * Copyright (c) 2016, 2017 Intel Corporation.
+ * Copyright 2016 Igor Kozhukhov <ikozhukhov@gmail.com>.
+ */
+
+/*
+ * Functions to convert between a list of vdevs and an nvlist representing the
+ * configuration.  Each entry in the list can be one of:
+ *
+ * 	Device vdevs
+ * 		disk=(path=..., devid=...)
+ * 		file=(path=...)
+ *
+ * 	Group vdevs
+ * 		raidz[1|2]=(...)
+ * 		mirror=(...)
+ *
+ * 	Hot spares
+ *
+ * While the underlying implementation supports it, group vdevs cannot contain
+ * other group vdevs.  All userland verification of devices is contained within
+ * this file.  If successful, the nvlist returned can be passed directly to the
+ * kernel; we've done as much verification as possible in userland.
+ *
+ * Hot spares are a special case, and passed down as an array of disk vdevs, at
+ * the same level as the root of the vdev tree.
+ *
+ * The only function exported by this file is 'make_root_vdev'.  The
+ * function performs several passes:
+ *
+ * 	1. Construct the vdev specification.  Performs syntax validation and
+ *         makes sure each device is valid.
+ * 	2. Check for devices in use.  Using libdiskmgt, makes sure that no
+ *         devices are also in use.  Some can be overridden using the 'force'
+ *         flag, others cannot.
+ * 	3. Check for replication errors if the 'force' flag is not specified.
+ *         validates that the replication level is consistent across the
+ *         entire pool.
+ * 	4. Call libzfs to label any whole disks with an EFI label.
+ */
+
+#include <assert.h>
+#include <errno.h>
+#include <fcntl.h>
+#include <libintl.h>
+#include <libnvpair.h>
+#include <libzutil.h>
+#include <limits.h>
+#include <sys/spa.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+#include <paths.h>
+#include <sys/stat.h>
+#include <sys/disk.h>
+#include <sys/mntent.h>
+#include <libgeom.h>
+
+#include "zpool_util.h"
+#include <sys/zfs_context.h>
+
+int
+check_device(const char *name, boolean_t force, boolean_t isspare,
+    boolean_t iswholedisk)
+{
+	char path[MAXPATHLEN];
+
+	if (strncmp(name, _PATH_DEV, sizeof (_PATH_DEV) - 1) != 0)
+		snprintf(path, sizeof (path), "%s%s", _PATH_DEV, name);
+	else
+		strlcpy(path, name, sizeof (path));
+
+	return (check_file(path, force, isspare));
+}
+
+#ifdef notyet
+/*
+ * By "whole disk" we mean an entire physical disk (something we can
+ * label, toggle the write cache on, etc.) as opposed to the full
+ * capacity of a pseudo-device such as lofi or did.  We act as if we
+ * are labeling the disk, which should be a pretty good test of whether
+ * it's a viable device or not.  Returns B_TRUE if it is and B_FALSE if
+ * it isn't.
+ */
+static boolean_t
+is_whole_disk(const char *arg)
+{
+	int fd;
+
+	fd = g_open(arg, 0);
+	if (fd >= 0) {
+		g_close(fd);
+		return (B_TRUE);
+	}
+	return (B_FALSE);
+}
+#endif
+
+boolean_t
+check_sector_size_database(char *path, int *sector_size)
+{
+	return (0);
+}
+
+void
+zpool_vdev_enable_file(struct stat64 *statbuf, boolean_t *wholedisk)
+{
+	if (S_ISCHR(statbuf->st_mode)) {
+		statbuf->st_mode &= ~S_IFCHR;
+		statbuf->st_mode |= S_IFBLK;
+		*wholedisk = B_FALSE;
+	}
+}

cmd/zpool/zpool.d/iostat

@@ -15,24 +15,32 @@ if [ "$1" = "-h" ] ; then
 	exit
 fi
 
+if [ "$(uname)" = "FreeBSD" ]; then
+	k="-K"
+	y=""
+else
+	k="-k"
+	y="-y"
+fi
+
 if [ "$script" = "iostat-1s" ] ; then
 	# Do a single one-second sample
 	extra="1 1"
 	# Don't show summary stats
-	y="-y"
+	brief="$y"
 elif [ "$script" = "iostat-10s" ] ; then
 	# Do a single ten-second sample
 	extra="10 1"
 	# Don't show summary stats
-	y="-y"
+	brief="$y"
 fi
 
 if [ -f "$VDEV_UPATH" ] ; then
 	# We're a file-based vdev, iostat doesn't work on us.  Do nothing.
 	exit
 fi
 
-out=$(eval "iostat $y -k -x $VDEV_UPATH $extra")
+out=$(eval "iostat -d $k $brief -x $VDEV_UPATH $extra" | tail -n 2)
 
 # Sample output (we want the last two lines):
 #
@@ -46,10 +54,10 @@ out=$(eval "iostat $y -k -x $VDEV_UPATH $extra")
 #
 
 # Get the column names
-cols=$(echo "$out" | grep Device)
+cols=$(echo "$out" | head -n 1)
 
 # Get the values and tab separate them to make them cut-able.
-vals="$(echo "$out" | grep -A1 Device | tail -n 1 | sed -r 's/[[:blank:]]+/\t/g')"
+vals=$(echo "$out" | tail -n 1 | sed -r 's/[[:blank:]]+/\t/g')
 
 i=0
 for col in $cols ; do

config/Rules.am

@@ -14,26 +14,51 @@ DEFAULT_INCLUDES += \
 	-I$(top_srcdir)/lib/libspl/include/os/linux
 endif
 
+if BUILD_FREEBSD
+DEFAULT_INCLUDES += \
+	-I$(top_srcdir)/lib/libspl/include/os/freebsd
+endif
+
 AM_LIBTOOLFLAGS = --silent
 
 AM_CFLAGS  = -std=gnu99 -Wall -Wstrict-prototypes -fno-strict-aliasing
 AM_CFLAGS += $(NO_OMIT_FRAME_POINTER)
 AM_CFLAGS += $(DEBUG_CFLAGS)
 AM_CFLAGS += $(ASAN_CFLAGS)
-AM_CFLAGS += $(CODE_COVERAGE_CFLAGS)
+AM_CFLAGS += $(CODE_COVERAGE_CFLAGS) $(NO_FORMAT_ZERO_LENGTH)
+if BUILD_FREEBSD
+AM_CFLAGS += -fPIC -Werror -Wno-unknown-pragmas -Wno-enum-conversion
+AM_CFLAGS += -include $(top_srcdir)/include/os/freebsd/spl/sys/ccompile.h
+AM_CFLAGS += -I/usr/include -I/usr/local/include
+AM_CFLAGS += -D_MACHINE_ENDIAN_H_
+endif
 
 AM_CPPFLAGS  = -D_GNU_SOURCE
 AM_CPPFLAGS += -D_REENTRANT
 AM_CPPFLAGS += -D_FILE_OFFSET_BITS=64
 AM_CPPFLAGS += -D_LARGEFILE64_SOURCE
 AM_CPPFLAGS += -DHAVE_LARGE_STACKS=1
-AM_CPPFLAGS += -DTEXT_DOMAIN=\"zfs-linux-user\"
 AM_CPPFLAGS += -DLIBEXECDIR=\"$(libexecdir)\"
 AM_CPPFLAGS += -DRUNSTATEDIR=\"$(runstatedir)\"
 AM_CPPFLAGS += -DSBINDIR=\"$(sbindir)\"
 AM_CPPFLAGS += -DSYSCONFDIR=\"$(sysconfdir)\"
 AM_CPPFLAGS += $(DEBUG_CPPFLAGS)
 AM_CPPFLAGS += $(CODE_COVERAGE_CPPFLAGS)
+if BUILD_LINUX
+AM_CPPFLAGS += -DTEXT_DOMAIN=\"zfs-linux-user\"
+endif
+if BUILD_FREEBSD
+AM_CPPFLAGS += -DTEXT_DOMAIN=\"zfs-freebsd-user\"
+# FIXME: are the rest of these needed?
+AM_CPPFLAGS += -D_SYS_VMEM_H_ -D_SYS_PROC_H_ -DVM_H -D_SYS_USER_H_
+AM_CPPFLAGS += -DZFS_DEBUG
+endif
 
 AM_LDFLAGS  = $(DEBUG_LDFLAGS)
 AM_LDFLAGS += $(ASAN_LDFLAGS)
+
+if BUILD_FREEBSD
+AM_LDFLAGS += -fstack-protector-strong -shared
+AM_LDFLAGS += -Wl,-x -Wl,--fatal-warnings -Wl,--warn-shared-textrel
+AM_LDFLAGS += -lm
+endif

config/always-arch.m4

@@ -3,7 +3,8 @@ dnl # Set the target arch for libspl atomic implementation and the icp
 dnl #
 AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_ARCH], [
 	AC_MSG_CHECKING(for target asm dir)
-	TARGET_ARCH=`echo ${target_cpu} | sed -e s/i.86/i386/`
+	subs="-e s/i.86/i386/ -e s/amd64/x86_64/"
+	TARGET_ARCH=$(echo ${target_cpu} | sed ${subs})
 
 	case $TARGET_ARCH in
 	i386|x86_64)

config/always-compiler-options.m4

@@ -52,7 +52,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_FRAME_LARGER_THAN], [
 	AC_MSG_CHECKING([whether $CC supports -Wframe-larger-than=<size>])
 
 	saved_flags="$CFLAGS"
-	CFLAGS="$CFLAGS -Wframe-larger-than=4096"
+	CFLAGS="$CFLAGS -Werror -Wframe-larger-than=4096"
 
 	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
 		FRAME_LARGER_THAN="-Wframe-larger-than=4096"
@@ -73,7 +73,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_TRUNCATION], [
 	AC_MSG_CHECKING([whether $CC supports -Wno-format-truncation])
 
 	saved_flags="$CFLAGS"
-	CFLAGS="$CFLAGS -Wno-format-truncation"
+	CFLAGS="$CFLAGS -Werror -Wno-format-truncation"
 
 	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
 		NO_FORMAT_TRUNCATION=-Wno-format-truncation
@@ -87,6 +87,27 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_TRUNCATION], [
 	AC_SUBST([NO_FORMAT_TRUNCATION])
 ])
 
+dnl #
+dnl # Check if gcc supports -Wno-format-truncation option.
+dnl #
+AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_FORMAT_ZERO_LENGTH], [
+	AC_MSG_CHECKING([whether $CC supports -Wno-format-zero-length])
+
+	saved_flags="$CFLAGS"
+	CFLAGS="$CFLAGS -Werror -Wno-format-zero-length"
+
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
+		NO_FORMAT_ZERO_LENGTH=-Wno-format-zero-length
+		AC_MSG_RESULT([yes])
+	], [
+		NO_FORMAT_ZERO_LENGTH=
+		AC_MSG_RESULT([no])
+	])
+
+	CFLAGS="$saved_flags"
+	AC_SUBST([NO_FORMAT_ZERO_LENGTH])
+])
+
 
 dnl #
 dnl # Check if gcc supports -Wno-bool-compare option.
@@ -100,7 +121,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_BOOL_COMPARE], [
 	AC_MSG_CHECKING([whether $CC supports -Wno-bool-compare])
 
 	saved_flags="$CFLAGS"
-	CFLAGS="$CFLAGS -Wbool-compare"
+	CFLAGS="$CFLAGS -Werror -Wbool-compare"
 
 	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
 		NO_BOOL_COMPARE=-Wno-bool-compare
@@ -126,7 +147,7 @@ AC_DEFUN([ZFS_AC_CONFIG_ALWAYS_CC_NO_UNUSED_BUT_SET_VARIABLE], [
 	AC_MSG_CHECKING([whether $CC supports -Wno-unused-but-set-variable])
 
 	saved_flags="$CFLAGS"
-	CFLAGS="$CFLAGS -Wunused-but-set-variable"
+	CFLAGS="$CFLAGS -Werror -Wunused-but-set-variable"
 
 	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([], [])], [
 		NO_UNUSED_BUT_SET_VARIABLE=-Wno-unused-but-set-variable