Add FreeBSD support to OpenZFS

Add the FreeBSD platform code to the OpenZFS repository.  As of this
commit the source can be compiled and tested on FreeBSD 11 and 12.
Subsequent commits are now required to compile on FreeBSD and Linux.
Additionally, they must pass the ZFS Test Suite on FreeBSD which is
being run by the CI.  As of this commit 1230 tests pass on FreeBSD
and there are no unexpected failures.

Reviewed-by: Sean Eric Fagan <sef@ixsystems.com>
Reviewed-by: Jorgen Lundman <lundman@lundman.net>
Reviewed-by: Richard Laager <rlaager@wiktel.com>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Co-authored-by: Ryan Moeller <ryan@iXsystems.com>
Signed-off-by: Matt Macy <mmacy@FreeBSD.org>
Signed-off-by: Ryan Moeller <ryan@iXsystems.com>
Closes #898 
Closes #8987

lib/libzutil/Makefile.am

@@ -21,13 +21,24 @@ USER_C += \
 	os/linux/zutil_compat.c
 endif
 
+if BUILD_FREEBSD
+USER_C += \
+	os/freebsd/zutil_device_path_os.c \
+	os/freebsd/zutil_import_os.c \
+	os/freebsd/zutil_compat.c
+endif
+
 nodist_libzutil_la_SOURCES = $(USER_C)
 
 libzutil_la_LIBADD = \
 	$(top_builddir)/lib/libavl/libavl.la \
-        $(top_builddir)/lib/libefi/libefi.la \
 	$(top_builddir)/lib/libtpool/libtpool.la
 
+if BUILD_LINUX
+libzutil_la_LIBADD += \
+        $(top_builddir)/lib/libefi/libefi.la
+endif
+
 libzutil_la_LIBADD += -lm $(LIBBLKID) $(LIBUDEV)
 
 EXTRA_DIST = $(USER_C)

lib/libzutil/os/freebsd/zutil_compat.c

@@ -0,0 +1,74 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+#include <sys/types.h>
+#include <sys/param.h>
+#include <sys/zfs_ioctl.h>
+#include <os/freebsd/zfs/sys/zfs_ioctl_compat.h>
+#include <libzutil.h>
+
+static int
+zcmd_ioctl_compat(int fd, int request, zfs_cmd_t *zc, const int cflag)
+{
+	int ret;
+	void *zc_c;
+	unsigned long ncmd;
+	zfs_iocparm_t zp;
+
+	switch (cflag) {
+	case ZFS_CMD_COMPAT_NONE:
+		ncmd = _IOWR('Z', request, zfs_iocparm_t);
+		zp.zfs_cmd = (uint64_t)zc;
+		zp.zfs_cmd_size = sizeof (zfs_cmd_t);
+		zp.zfs_ioctl_version = ZFS_IOCVER_ZOF;
+		return (ioctl(fd, ncmd, &zp));
+	default:
+		abort();
+		return (EINVAL);
+	}
+
+	ret = ioctl(fd, ncmd, zc_c);
+	zfs_cmd_compat_get(zc, (caddr_t)zc_c, cflag);
+	free(zc_c);
+
+	return (ret);
+}
+
+/*
+ * This is FreeBSD version of ioctl, because Solaris' ioctl() updates
+ * zc_nvlist_dst_size even if an error is returned, on FreeBSD if an
+ * error is returned zc_nvlist_dst_size won't be updated.
+ */
+int
+zfs_ioctl_fd(int fd, unsigned long request, zfs_cmd_t *zc)
+{
+	size_t oldsize;
+	int ret, cflag = ZFS_CMD_COMPAT_NONE;
+
+	oldsize = zc->zc_nvlist_dst_size;
+	ret = zcmd_ioctl_compat(fd, request, zc, cflag);
+
+	if (ret == 0 && oldsize < zc->zc_nvlist_dst_size) {
+		ret = -1;
+		errno = ENOMEM;
+	}
+
+	return (ret);
+}

lib/libzutil/os/freebsd/zutil_device_path_os.c

@@ -0,0 +1,132 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+#include <ctype.h>
+#include <fcntl.h>
+#include <paths.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include <sys/stat.h>
+
+#include <libgeom.h>
+
+#include <libzutil.h>
+
+/*
+ * We don't strip/append partitions on FreeBSD.
+ */
+
+/*
+ * Note: The caller must free the returned string.
+ */
+char *
+zfs_strip_partition(char *dev)
+{
+	return (strdup(dev));
+}
+
+int
+zfs_append_partition(char *path, size_t max_len)
+{
+	return (strnlen(path, max_len));
+}
+
+/*
+ * Strip the path from a device name.
+ * On FreeBSD we only want to remove "/dev/" from the beginning of
+ * paths if present.
+ */
+char *
+zfs_strip_path(char *path)
+{
+	if (strncmp(path, _PATH_DEV, sizeof (_PATH_DEV) - 1) == 0)
+		return (path + sizeof (_PATH_DEV) - 1);
+	else
+		return (path);
+}
+
+char *
+zfs_get_underlying_path(const char *dev_name)
+{
+
+	if (dev_name == NULL)
+		return (NULL);
+
+	return (realpath(dev_name, NULL));
+}
+
+boolean_t
+zfs_dev_is_whole_disk(const char *dev_name)
+{
+	int fd;
+
+	fd = g_open(dev_name, 0);
+	if (fd >= 0) {
+		g_close(fd);
+		return (B_TRUE);
+	}
+	return (B_FALSE);
+}
+
+/*
+ * Wait up to timeout_ms for udev to set up the device node.  The device is
+ * considered ready when libudev determines it has been initialized, all of
+ * the device links have been verified to exist, and it has been allowed to
+ * settle.  At this point the device the device can be accessed reliably.
+ * Depending on the complexity of the udev rules this process could take
+ * several seconds.
+ */
+int
+zpool_label_disk_wait(const char *path, int timeout_ms)
+{
+	int settle_ms = 50;
+	long sleep_ms = 10;
+	hrtime_t start, settle;
+	struct stat64 statbuf;
+
+	start = gethrtime();
+	settle = 0;
+
+	do {
+		errno = 0;
+		if ((stat64(path, &statbuf) == 0) && (errno == 0)) {
+			if (settle == 0)
+				settle = gethrtime();
+			else if (NSEC2MSEC(gethrtime() - settle) >= settle_ms)
+				return (0);
+		} else if (errno != ENOENT) {
+			return (errno);
+		}
+
+		usleep(sleep_ms * MILLISEC);
+	} while (NSEC2MSEC(gethrtime() - start) < timeout_ms);
+
+	return (ENODEV);
+}
+
+/* ARGSUSED */
+boolean_t
+is_mpath_whole_disk(const char *path)
+{
+	return (B_FALSE);
+}

lib/libzutil/os/freebsd/zutil_import_os.c

@@ -0,0 +1,239 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+
+/*
+ * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2012, 2017 by Delphix. All rights reserved.
+ * Copyright 2015 RackTop Systems.
+ * Copyright 2016 Nexenta Systems, Inc.
+ */
+
+/*
+ * Pool import support functions.
+ *
+ * To import a pool, we rely on reading the configuration information from the
+ * ZFS label of each device.  If we successfully read the label, then we
+ * organize the configuration information in the following hierarchy:
+ *
+ *	pool guid -> toplevel vdev guid -> label txg
+ *
+ * Duplicate entries matching this same tuple will be discarded.  Once we have
+ * examined every device, we pick the best label txg config for each toplevel
+ * vdev.  We then arrange these toplevel vdevs into a complete pool config, and
+ * update any paths that have changed.  Finally, we attempt to import the pool
+ * using our derived config, and record the results.
+ */
+
+#include <aio.h>
+#include <ctype.h>
+#include <dirent.h>
+#include <errno.h>
+#include <libintl.h>
+#include <libgen.h>
+#include <stddef.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/disk.h>
+#include <sys/ioctl.h>
+#include <sys/stat.h>
+#include <unistd.h>
+#include <fcntl.h>
+
+#include <sys/efi_partition.h>
+#include <thread_pool.h>
+#include <libgeom.h>
+
+#include <sys/vdev_impl.h>
+
+#include <libzutil.h>
+
+#include "zutil_import.h"
+
+/*
+ * Update a leaf vdev's persistent device strings
+ *
+ * - only applies for a dedicated leaf vdev (aka whole disk)
+ * - updated during pool create|add|attach|import
+ * - used for matching device matching during auto-{online,expand,replace}
+ * - stored in a leaf disk config label (i.e. alongside 'path' NVP)
+ * - these strings are currently not used in kernel (i.e. for vdev_disk_open)
+ *
+ * On FreeBSD we currently just strip devid and phys_path to avoid confusion.
+ */
+void
+update_vdev_config_dev_strs(nvlist_t *nv)
+{
+	(void) nvlist_remove_all(nv, ZPOOL_CONFIG_DEVID);
+	(void) nvlist_remove_all(nv, ZPOOL_CONFIG_PHYS_PATH);
+}
+
+/*
+ * Do not even look at these devices.
+ */
+static const char * const blacklist_devs[] = {
+	"nfslock",
+	"sequencer",
+	"zfs",
+};
+#define	BLACKLIST_DIR		"/dev/"
+#define	BLACKLIST_DIR_LEN	5
+
+void
+zpool_open_func(void *arg)
+{
+	rdsk_node_t *rn = arg;
+	struct stat64 statbuf;
+	nvlist_t *config;
+	size_t i;
+	int num_labels;
+	int fd;
+	off_t mediasize = 0;
+
+	/*
+	 * Do not even look at blacklisted devices.
+	 */
+	if (strncmp(rn->rn_name, BLACKLIST_DIR, BLACKLIST_DIR_LEN) == 0) {
+		char *name = rn->rn_name + BLACKLIST_DIR_LEN;
+		for (i = 0; i < nitems(blacklist_devs); ++i) {
+			const char *badname = blacklist_devs[i];
+			size_t len = strlen(badname);
+			if (strncmp(name, badname, len) == 0) {
+				return;
+			}
+		}
+	}
+
+	/*
+	 * O_NONBLOCK so we don't hang trying to open things like serial ports.
+	 */
+	if ((fd = open(rn->rn_name, O_RDONLY|O_NONBLOCK)) < 0)
+		return;
+
+	/*
+	 * Ignore failed stats.
+	 */
+	if (fstat64(fd, &statbuf) != 0)
+		goto out;
+	/*
+	 * We only want regular files, character devs and block devs.
+	 */
+	if (S_ISREG(statbuf.st_mode)) {
+		/* Check if this file is too small to hold a zpool. */
+		if (statbuf.st_size < SPA_MINDEVSIZE) {
+			goto out;
+		}
+	} else if (S_ISCHR(statbuf.st_mode) || S_ISBLK(statbuf.st_mode)) {
+		/* Check if this device is too small to hold a zpool. */
+		if (ioctl(fd, DIOCGMEDIASIZE, &mediasize) != 0 ||
+		    mediasize < SPA_MINDEVSIZE) {
+			goto out;
+		}
+	} else {
+		goto out;
+	}
+
+	if (zpool_read_label(fd, &config, &num_labels) != 0)
+		goto out;
+	if (num_labels == 0) {
+		nvlist_free(config);
+		goto out;
+	}
+
+	rn->rn_config = config;
+	rn->rn_num_labels = num_labels;
+
+	/* TODO: Reuse labelpaths logic from Linux? */
+out:
+	(void) close(fd);
+}
+
+static const char *
+zpool_default_import_path[] = {
+	"/dev"
+};
+
+const char * const *
+zpool_default_search_paths(size_t *count)
+{
+	*count = nitems(zpool_default_import_path);
+	return (zpool_default_import_path);
+}
+
+int
+zpool_find_import_blkid(libpc_handle_t *hdl, pthread_mutex_t *lock,
+    avl_tree_t **slice_cache)
+{
+	char *end, path[MAXPATHLEN];
+	rdsk_node_t *slice;
+	struct gmesh mesh;
+	struct gclass *mp;
+	struct ggeom *gp;
+	struct gprovider *pp;
+	avl_index_t where;
+	size_t pathleft;
+	int error;
+
+	end = stpcpy(path, "/dev/");
+	pathleft = &path[sizeof (path)] - end;
+
+	error = geom_gettree(&mesh);
+	if (error != 0)
+		return (error);
+
+	*slice_cache = zutil_alloc(hdl, sizeof (avl_tree_t));
+	avl_create(*slice_cache, slice_cache_compare, sizeof (rdsk_node_t),
+	    offsetof(rdsk_node_t, rn_node));
+
+	LIST_FOREACH(mp, &mesh.lg_class, lg_class) {
+		LIST_FOREACH(gp, &mp->lg_geom, lg_geom) {
+			LIST_FOREACH(pp, &gp->lg_provider, lg_provider) {
+				strlcpy(end, pp->lg_name, pathleft);
+				slice = zutil_alloc(hdl, sizeof (rdsk_node_t));
+				slice->rn_name = zutil_strdup(hdl, path);
+				slice->rn_vdev_guid = 0;
+				slice->rn_lock = lock;
+				slice->rn_avl = *slice_cache;
+				slice->rn_hdl = hdl;
+				slice->rn_labelpaths = B_FALSE;
+				slice->rn_order = IMPORT_ORDER_DEFAULT;
+
+				pthread_mutex_lock(lock);
+				if (avl_find(*slice_cache, slice, &where)) {
+					free(slice->rn_name);
+					free(slice);
+				} else {
+					avl_insert(*slice_cache, slice, where);
+				}
+				pthread_mutex_unlock(lock);
+			}
+		}
+	}
+
+	geom_deletetree(&mesh);
+
+	return (0);
+}
+
+int
+zfs_dev_flush(int fd __unused)
+{
+	return (0);
+}