zfs label bootenv should store data as nvlist

nvlist does allow us to support different data types and systems.

To encapsulate user data to/from nvlist, the libzfsbootenv library is
provided.

Reviewed-by: Arvind Sankar <nivedita@alum.mit.edu>
Reviewed-by: Allan Jude <allan@klarasystems.com>
Reviewed-by: Paul Dagnelie <pcd@delphix.com>
Reviewed-by: Igor Kozhukhov <igor@dilos.org>
Signed-off-by: Toomas Soome <tsoome@me.com>
Closes #10774

configure.ac

@@ -161,6 +161,8 @@ AC_CONFIG_FILES([
 	lib/libuutil/Makefile
 	lib/libzfs/Makefile
 	lib/libzfs/libzfs.pc
+	lib/libzfsbootenv/Makefile
+	lib/libzfsbootenv/libzfsbootenv.pc
 	lib/libzfs_core/Makefile
 	lib/libzfs_core/libzfs_core.pc
 	lib/libzpool/Makefile

include/Makefile.am

@@ -15,6 +15,7 @@ USER_H = \
 	libuutil.h \
 	libuutil_impl.h \
 	libzfs.h \
+	libzfsbootenv.h \
 	libzfs_core.h \
 	libzfs_impl.h \
 	libzutil.h \

include/libzfs.h

@@ -892,8 +892,8 @@ extern int zpool_in_use(libzfs_handle_t *, int, pool_state_t *, char **,
  * Label manipulation.
  */
 extern int zpool_clear_label(int);
-extern int zpool_set_bootenv(zpool_handle_t *, const char *);
-extern int zpool_get_bootenv(zpool_handle_t *, char *, size_t, off_t);
+extern int zpool_set_bootenv(zpool_handle_t *, const nvlist_t *);
+extern int zpool_get_bootenv(zpool_handle_t *, nvlist_t **);
 
 /*
  * Management interfaces for SMB ACL files

include/libzfs_core.h

@@ -135,7 +135,7 @@ int lzc_wait(const char *, zpool_wait_activity_t, boolean_t *);
 int lzc_wait_tag(const char *, zpool_wait_activity_t, uint64_t, boolean_t *);
 int lzc_wait_fs(const char *, zfs_wait_activity_t, boolean_t *);
 
-int lzc_set_bootenv(const char *, const char *);
+int lzc_set_bootenv(const char *, const nvlist_t *);
 int lzc_get_bootenv(const char *, nvlist_t **);
 #ifdef	__cplusplus
 }

include/libzfsbootenv.h

@@ -0,0 +1,41 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2020 Toomas Soome <tsoome@me.com>
+ */
+
+#ifndef _LIBZFSBOOTENV_H
+#define	_LIBZFSBOOTENV_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef enum lzbe_flags {
+	lzbe_add,	/* add data to existing nvlist */
+	lzbe_replace	/* replace current nvlist */
+} lzbe_flags_t;
+
+extern int lzbe_nvlist_get(const char *, const char *, void **);
+extern int lzbe_nvlist_set(const char *, const char *, void *);
+extern void lzbe_nvlist_free(void *);
+extern int lzbe_add_pair(void *, const char *, const char *, void *, size_t);
+extern int lzbe_remove_pair(void *, const char *);
+extern int lzbe_set_boot_device(const char *, lzbe_flags_t, const char *);
+extern int lzbe_get_boot_device(const char *, char **);
+extern int lzbe_bootenv_print(const char *, const char *, FILE *);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _LIBZFSBOOTENV_H */

include/os/freebsd/zfs/sys/Makefile.am

@@ -2,6 +2,7 @@ KERNEL_H = \
 	freebsd_crypto.h \
 	sha2.h \
 	vdev_os.h \
+	zfs_bootenv_os.h \
 	zfs_context_os.h \
 	zfs_ctldir.h \
 	zfs_dir.h \

include/os/freebsd/zfs/sys/zfs_bootenv_os.h

@@ -0,0 +1,29 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2020 Toomas Soome <tsoome@me.com>
+ */
+
+#ifndef _ZFS_BOOTENV_OS_H
+#define	_ZFS_BOOTENV_OS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define	BOOTENV_OS		BE_FREEBSD_VENDOR
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _ZFS_BOOTENV_OS_H */

include/os/linux/zfs/sys/Makefile.am

@@ -16,6 +16,7 @@ KERNEL_H = \
 	trace_zil.h \
 	trace_zio.h \
 	trace_zrlock.h \
+	zfs_bootenv_os.h \
 	zfs_context_os.h \
 	zfs_ctldir.h \
 	zfs_dir.h \

include/os/linux/zfs/sys/zfs_bootenv_os.h

@@ -0,0 +1,29 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2020 Toomas Soome <tsoome@me.com>
+ */
+
+#ifndef _ZFS_BOOTENV_OS_H
+#define	_ZFS_BOOTENV_OS_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define	BOOTENV_OS		BE_LINUX_VENDOR
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _ZFS_BOOTENV_OS_H */

include/sys/Makefile.am

@@ -102,6 +102,7 @@ COMMON_H = \
 	zcp_set.h \
 	zfeature.h \
 	zfs_acl.h \
+	zfs_bootenv.h \
 	zfs_context.h \
 	zfs_debug.h \
 	zfs_delay.h \

include/sys/fs/zfs.h

@@ -1336,8 +1336,8 @@ typedef enum zfs_ioc {
 	ZFS_IOC_NEXTBOOT,			/* 0x84 (FreeBSD) */
 	ZFS_IOC_JAIL,				/* 0x85 (FreeBSD) */
 	ZFS_IOC_UNJAIL,				/* 0x86 (FreeBSD) */
-	ZFS_IOC_SET_BOOTENV,			/* 0x87 (Linux) */
-	ZFS_IOC_GET_BOOTENV,			/* 0x88 (Linux) */
+	ZFS_IOC_SET_BOOTENV,			/* 0x87 */
+	ZFS_IOC_GET_BOOTENV,			/* 0x88 */
 	ZFS_IOC_LAST
 } zfs_ioc_t;
 

include/sys/vdev.h

@@ -181,7 +181,7 @@ extern void vdev_config_generate_stats(vdev_t *vd, nvlist_t *nv);
 extern void vdev_label_write(zio_t *zio, vdev_t *vd, int l, abd_t *buf, uint64_t
     offset, uint64_t size, zio_done_func_t *done, void *priv, int flags);
 extern int vdev_label_read_bootenv(vdev_t *, nvlist_t *);
-extern int vdev_label_write_bootenv(vdev_t *, char *);
+extern int vdev_label_write_bootenv(vdev_t *, nvlist_t *);
 
 typedef enum {
 	VDEV_LABEL_CREATE,	/* create/add a new device */

include/sys/vdev_impl.h

@@ -476,7 +476,16 @@ typedef struct vdev_phys {
 } vdev_phys_t;
 
 typedef enum vbe_vers {
-	/* The bootenv file is stored as ascii text in the envblock */
+	/*
+	 * The bootenv file is stored as ascii text in the envblock.
+	 * It is used by the GRUB bootloader used on Linux to store the
+	 * contents of the grubenv file. The file is stored as raw ASCII,
+	 * and is protected by an embedded checksum. By default, GRUB will
+	 * check if the boot filesystem supports storing the environment data
+	 * in a special location, and if so, will invoke filesystem specific
+	 * logic to retrieve it. This can be overriden by a variable, should
+	 * the user so desire.
+	 */
 	VB_RAW = 0,
 
 	/*

include/sys/zfs_bootenv.h

@@ -0,0 +1,53 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+
+/*
+ * Copyright 2020 Toomas Soome <tsoome@me.com>
+ */
+
+#ifndef _ZFS_BOOTENV_H
+#define	_ZFS_BOOTENV_H
+
+/*
+ * Define macros for label bootenv nvlist pair keys.
+ */
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define	BOOTENV_VERSION		"version"
+
+#define	BE_ILLUMOS_VENDOR	"illumos"
+#define	BE_FREEBSD_VENDOR	"freebsd"
+#define	BE_GRUB_VENDOR		"grub"
+#define	BE_LINUX_VENDOR		"linux"
+
+#include <sys/zfs_bootenv_os.h>
+
+#define	GRUB_ENVMAP		BE_GRUB_VENDOR ":" "envmap"
+
+#define	FREEBSD_BOOTONCE	BE_FREEBSD_VENDOR ":" "bootonce"
+#define	FREEBSD_BOOTONCE_USED	BE_FREEBSD_VENDOR ":" "bootonce-used"
+#define	FREEBSD_NVSTORE		BE_FREEBSD_VENDOR ":" "nvstore"
+#define	ILLUMOS_BOOTONCE	BE_ILLUMOS_VENDOR ":" "bootonce"
+#define	ILLUMOS_BOOTONCE_USED	BE_ILLUMOS_VENDOR ":" "bootonce-used"
+#define	ILLUMOS_NVSTORE		BE_ILLUMOS_VENDOR ":" "nvstore"
+
+#define	OS_BOOTONCE		BOOTENV_OS ":" "bootonce"
+#define	OS_BOOTONCE_USED	BOOTENV_OS ":" "bootonce-used"
+#define	OS_NVSTORE		BOOTENV_OS ":" "nvstore"
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif /* _ZFS_BOOTENV_H */

lib/Makefile.am

@@ -13,6 +13,6 @@ SUBDIRS += libnvpair
 # libzutil depends on libefi if present
 SUBDIRS += libzutil libunicode
 
-# These four libraries, which are installed as the final build product,
+# These five libraries, which are installed as the final build product,
 # incorporate the eight convenience libraries given above.
-SUBDIRS += libuutil libzfs_core libzfs libzpool
+SUBDIRS += libuutil libzfs_core libzfs libzpool libzfsbootenv

lib/libzfs/libzfs_pool.c

@@ -4495,7 +4495,7 @@ zpool_wait_status(zpool_handle_t *zhp, zpool_wait_activity_t activity,
 }
 
 int
-zpool_set_bootenv(zpool_handle_t *zhp, const char *envmap)
+zpool_set_bootenv(zpool_handle_t *zhp, const nvlist_t *envmap)
 {
 	int error = lzc_set_bootenv(zhp->zpool_name, envmap);
 	if (error != 0) {
@@ -4508,24 +4508,20 @@ zpool_set_bootenv(zpool_handle_t *zhp, const char *envmap)
 }
 
 int
-zpool_get_bootenv(zpool_handle_t *zhp, char *outbuf, size_t size, off_t offset)
+zpool_get_bootenv(zpool_handle_t *zhp, nvlist_t **nvlp)
 {
-	nvlist_t *nvl = NULL;
-	int error = lzc_get_bootenv(zhp->zpool_name, &nvl);
+	nvlist_t *nvl;
+	int error;
+
+	nvl = NULL;
+	error = lzc_get_bootenv(zhp->zpool_name, &nvl);
 	if (error != 0) {
 		(void) zpool_standard_error_fmt(zhp->zpool_hdl, error,
 		    dgettext(TEXT_DOMAIN,
 		    "error getting bootenv in pool '%s'"), zhp->zpool_name);
-		return (-1);
-	}
-	char *envmap = fnvlist_lookup_string(nvl, "envmap");
-	if (offset >= strlen(envmap)) {
-		fnvlist_free(nvl);
-		return (0);
+	} else {
+		*nvlp = nvl;
 	}
 
-	strncpy(outbuf, envmap + offset, size);
-	int bytes = MIN(strlen(envmap + offset), size);
-	fnvlist_free(nvl);
-	return (bytes);
+	return (error);
 }

lib/libzfs_core/libzfs_core.c

@@ -1625,13 +1625,9 @@ lzc_wait_fs(const char *fs, zfs_wait_activity_t activity, boolean_t *waited)
  * Set the bootenv contents for the given pool.
  */
 int
-lzc_set_bootenv(const char *pool, const char *env)
+lzc_set_bootenv(const char *pool, const nvlist_t *env)
 {
-	nvlist_t *args = fnvlist_alloc();
-	fnvlist_add_string(args, "envmap", env);
-	int error = lzc_ioctl(ZFS_IOC_SET_BOOTENV, pool, args, NULL);
-	fnvlist_free(args);
-	return (error);
+	return (lzc_ioctl(ZFS_IOC_SET_BOOTENV, pool, (nvlist_t *)env, NULL));
 }
 
 /*

lib/libzfsbootenv/.gitignore

@@ -0,0 +1 @@
+/libzfsbootenv.pc

lib/libzfsbootenv/Makefile.am

@@ -0,0 +1,32 @@
+include $(top_srcdir)/config/Rules.am
+
+pkgconfig_DATA = libzfsbootenv.pc
+
+lib_LTLIBRARIES = libzfsbootenv.la
+
+if BUILD_FREEBSD
+DEFAULT_INCLUDES += -I$(top_srcdir)/include/os/freebsd/zfs
+endif
+if BUILD_LINUX
+DEFAULT_INCLUDES += -I$(top_srcdir)/include/os/linux/zfs
+endif
+
+USER_C = \
+	lzbe_device.c \
+	lzbe_pair.c \
+	lzbe_util.c
+
+dist_libzfsbootenv_la_SOURCES = \
+	$(USER_C)
+
+libzfsbootenv_la_LIBADD = \
+	$(abs_top_builddir)/lib/libzfs/libzfs.la \
+	$(abs_top_builddir)/lib/libnvpair/libnvpair.la
+
+libzfsbootenv_la_LDFLAGS =
+
+if !ASAN_ENABLED
+libzfsbootenv_la_LDFLAGS += -Wl,-z,defs
+endif
+
+libzfsbootenv_la_LDFLAGS += -version-info 1:0:0

lib/libzfsbootenv/libzfsbootenv.pc.in

@@ -0,0 +1,12 @@
+prefix=@prefix@
+exec_prefix=@exec_prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: libzfsbootenv
+Description: LibZFSBootENV library
+Version: @VERSION@
+URL: https://zfsonlinux.org
+Requires: libzfs libnvpair
+Cflags: -I${includedir}
+Libs: -L${libdir} -lzfsbootenv

lib/libzfsbootenv/lzbe_device.c

@@ -0,0 +1,164 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+/*
+ * Copyright 2020 Toomas Soome <tsoome@me.com>
+ */
+
+#include <sys/types.h>
+#include <string.h>
+#include <libzfs.h>
+#include <libzfsbootenv.h>
+#include <sys/zfs_bootenv.h>
+#include <sys/vdev_impl.h>
+
+/*
+ * Store device name to zpool label bootenv area.
+ * This call will set bootenv version to VB_NVLIST, if bootenv currently
+ * does contain other version, then old data will be replaced.
+ */
+int
+lzbe_set_boot_device(const char *pool, lzbe_flags_t flag, const char *device)
+{
+	libzfs_handle_t *hdl;
+	zpool_handle_t *zphdl;
+	nvlist_t *nv;
+	char *descriptor;
+	uint64_t version;
+	int rv = -1;
+
+	if (pool == NULL || *pool == '\0')
+		return (rv);
+
+	if ((hdl = libzfs_init()) == NULL)
+		return (rv);
+
+	zphdl = zpool_open(hdl, pool);
+	if (zphdl == NULL) {
+		libzfs_fini(hdl);
+		return (rv);
+	}
+
+	switch (flag) {
+	case lzbe_add:
+		rv = zpool_get_bootenv(zphdl, &nv);
+		if (rv == 0) {
+			/*
+			 * We got the nvlist, check for version.
+			 * if version is missing or is not VB_NVLIST,
+			 * create new list.
+			 */
+			rv = nvlist_lookup_uint64(nv, BOOTENV_VERSION,
+			    &version);
+			if (rv == 0 && version == VB_NVLIST)
+				break;
+
+			/* Drop this nvlist */
+			fnvlist_free(nv);
+		}
+		/* FALLTHROUGH */
+	case lzbe_replace:
+		nv = fnvlist_alloc();
+		break;
+	default:
+		return (rv);
+	}
+
+	/* version is mandatory */
+	fnvlist_add_uint64(nv, BOOTENV_VERSION, VB_NVLIST);
+
+	/*
+	 * If device name is empty, remove boot device configuration.
+	 */
+	if ((device == NULL || *device == '\0')) {
+		if (nvlist_exists(nv, OS_BOOTONCE))
+			fnvlist_remove(nv, OS_BOOTONCE);
+	} else {
+		/*
+		 * Use device name directly if it does start with
+		 * prefix "zfs:". Otherwise, add prefix and sufix.
+		 */
+		if (strncmp(device, "zfs:", 4) == 0) {
+			fnvlist_add_string(nv, OS_BOOTONCE, device);
+		} else {
+			descriptor = NULL;
+			if (asprintf(&descriptor, "zfs:%s:", device) > 0)
+				fnvlist_add_string(nv, OS_BOOTONCE, descriptor);
+			else
+				rv = ENOMEM;
+			free(descriptor);
+		}
+	}
+
+	rv = zpool_set_bootenv(zphdl, nv);
+	if (rv != 0)
+		fprintf(stderr, "%s\n", libzfs_error_description(hdl));
+
+	fnvlist_free(nv);
+	zpool_close(zphdl);
+	libzfs_fini(hdl);
+	return (rv);
+}
+
+/*
+ * Return boot device name from bootenv, if set.
+ */
+int
+lzbe_get_boot_device(const char *pool, char **device)
+{
+	libzfs_handle_t *hdl;
+	zpool_handle_t *zphdl;
+	nvlist_t *nv;
+	char *val;
+	int rv = -1;
+
+	if (pool == NULL || *pool == '\0' || device == NULL)
+		return (rv);
+
+	if ((hdl = libzfs_init()) == NULL)
+		return (rv);
+
+	zphdl = zpool_open(hdl, pool);
+	if (zphdl == NULL) {
+		libzfs_fini(hdl);
+		return (rv);
+	}
+
+	rv = zpool_get_bootenv(zphdl, &nv);
+	if (rv == 0) {
+		rv = nvlist_lookup_string(nv, OS_BOOTONCE, &val);
+		if (rv == 0) {
+			/*
+			 * zfs device descriptor is in form of "zfs:dataset:",
+			 * we only do need dataset name.
+			 */
+			if (strncmp(val, "zfs:", 4) == 0) {
+				val += 4;
+				val = strdup(val);
+				if (val != NULL) {
+					size_t len = strlen(val);
+
+					if (val[len - 1] == ':')
+						val[len - 1] = '\0';
+					*device = val;
+				} else {
+					rv = ENOMEM;
+				}
+			} else {
+				rv = EINVAL;
+			}
+		}
+		nvlist_free(nv);
+	}
+
+	zpool_close(zphdl);
+	libzfs_fini(hdl);
+	return (rv);
+}

lib/libzfsbootenv/lzbe_pair.c

@@ -0,0 +1,329 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+/*
+ * Copyright 2020 Toomas Soome <tsoome@me.com>
+ */
+
+#include <sys/types.h>
+#include <string.h>
+#include <libzfs.h>
+#include <libzfsbootenv.h>
+
+/*
+ * Get or create nvlist. If key is not NULL, get nvlist from bootenv,
+ * otherwise return bootenv.
+ */
+int
+lzbe_nvlist_get(const char *pool, const char *key, void **ptr)
+{
+	libzfs_handle_t *hdl;
+	zpool_handle_t *zphdl;
+	nvlist_t *nv;
+	int rv = -1;
+
+	if (pool == NULL || *pool == '\0')
+		return (rv);
+
+	if ((hdl = libzfs_init()) == NULL) {
+		return (rv);
+	}
+
+	zphdl = zpool_open(hdl, pool);
+	if (zphdl == NULL) {
+		libzfs_fini(hdl);
+		return (rv);
+	}
+
+	rv = zpool_get_bootenv(zphdl, &nv);
+	if (rv == 0) {
+		nvlist_t *nvl, *dup;
+
+		if (key != NULL) {
+			rv = nvlist_lookup_nvlist(nv, key, &nvl);
+			if (rv == 0) {
+				rv = nvlist_dup(nvl, &dup, 0);
+				nvlist_free(nv);
+				if (rv == 0)
+					nv = dup;
+				else
+					nv = NULL;
+			} else {
+				nvlist_free(nv);
+				rv = nvlist_alloc(&nv, NV_UNIQUE_NAME, 0);
+			}
+		}
+		*ptr = nv;
+	}
+
+	zpool_close(zphdl);
+	libzfs_fini(hdl);
+	return (rv);
+}
+
+int
+lzbe_nvlist_set(const char *pool, const char *key, void *ptr)
+{
+	libzfs_handle_t *hdl;
+	zpool_handle_t *zphdl;
+	nvlist_t *nv;
+	int rv = -1;
+
+	if (pool == NULL || *pool == '\0')
+		return (rv);
+
+	if ((hdl = libzfs_init()) == NULL) {
+		return (rv);
+	}
+
+	zphdl = zpool_open(hdl, pool);
+	if (zphdl == NULL) {
+		libzfs_fini(hdl);
+		return (rv);
+	}
+
+	if (key != NULL) {
+		rv = zpool_get_bootenv(zphdl, &nv);
+		if (rv == 0) {
+			rv = nvlist_add_nvlist(nv, key, ptr);
+			if (rv == 0)
+				rv = zpool_set_bootenv(zphdl, nv);
+			nvlist_free(nv);
+		}
+	} else {
+		rv = zpool_set_bootenv(zphdl, ptr);
+	}
+
+	zpool_close(zphdl);
+	libzfs_fini(hdl);
+	return (rv);
+}
+
+/*
+ * free nvlist we got via lzbe_nvlist_get()
+ */
+void
+lzbe_nvlist_free(void *ptr)
+{
+	nvlist_free(ptr);
+}
+
+static const char *typenames[] = {
+	"DATA_TYPE_UNKNOWN",
+	"DATA_TYPE_BOOLEAN",
+	"DATA_TYPE_BYTE",
+	"DATA_TYPE_INT16",
+	"DATA_TYPE_UINT16",
+	"DATA_TYPE_INT32",
+	"DATA_TYPE_UINT32",
+	"DATA_TYPE_INT64",
+	"DATA_TYPE_UINT64",
+	"DATA_TYPE_STRING",
+	"DATA_TYPE_BYTE_ARRAY",
+	"DATA_TYPE_INT16_ARRAY",
+	"DATA_TYPE_UINT16_ARRAY",
+	"DATA_TYPE_INT32_ARRAY",
+	"DATA_TYPE_UINT32_ARRAY",
+	"DATA_TYPE_INT64_ARRAY",
+	"DATA_TYPE_UINT64_ARRAY",
+	"DATA_TYPE_STRING_ARRAY",
+	"DATA_TYPE_HRTIME",
+	"DATA_TYPE_NVLIST",
+	"DATA_TYPE_NVLIST_ARRAY",
+	"DATA_TYPE_BOOLEAN_VALUE",
+	"DATA_TYPE_INT8",
+	"DATA_TYPE_UINT8",
+	"DATA_TYPE_BOOLEAN_ARRAY",
+	"DATA_TYPE_INT8_ARRAY",
+	"DATA_TYPE_UINT8_ARRAY"
+};
+
+static int
+nvpair_type_from_name(const char *name)
+{
+	unsigned i;
+
+	for (i = 0; i < ARRAY_SIZE(typenames); i++) {
+		if (strcmp(name, typenames[i]) == 0)
+			return (i);
+	}
+	return (0);
+}
+
+/*
+ * Add pair defined by key, type and value into nvlist.
+ */
+int
+lzbe_add_pair(void *ptr, const char *key, const char *type, void *value,
+    size_t size)
+{
+	nvlist_t *nv = ptr;
+	data_type_t dt;
+	int rv = 0;
+
+	if (ptr == NULL || key == NULL || value == NULL)
+		return (rv);
+
+	if (type == NULL)
+		type = "DATA_TYPE_STRING";
+	dt = nvpair_type_from_name(type);
+	if (dt == DATA_TYPE_UNKNOWN)
+		return (EINVAL);
+
+	switch (dt) {
+	case DATA_TYPE_BYTE:
+		if (size != sizeof (uint8_t)) {
+			rv = EINVAL;
+			break;
+		}
+		rv = nvlist_add_byte(nv, key, *(uint8_t *)value);
+		break;
+
+	case DATA_TYPE_INT16:
+		if (size != sizeof (int16_t)) {
+			rv = EINVAL;
+			break;
+		}
+		rv = nvlist_add_int16(nv, key, *(int16_t *)value);
+		break;
+
+	case DATA_TYPE_UINT16:
+		if (size != sizeof (uint16_t)) {
+			rv = EINVAL;
+			break;
+		}
+		rv = nvlist_add_uint16(nv, key, *(uint16_t *)value);
+		break;
+
+	case DATA_TYPE_INT32:
+		if (size != sizeof (int32_t)) {
+			rv = EINVAL;
+			break;
+		}
+		rv = nvlist_add_int32(nv, key, *(int32_t *)value);
+		break;
+
+	case DATA_TYPE_UINT32:
+		if (size != sizeof (uint32_t)) {
+			rv = EINVAL;
+			break;
+		}
+		rv = nvlist_add_uint32(nv, key, *(uint32_t *)value);
+		break;
+
+	case DATA_TYPE_INT64:
+		if (size != sizeof (int64_t)) {
+			rv = EINVAL;
+			break;
+		}
+		rv = nvlist_add_int64(nv, key, *(int64_t *)value);
+		break;
+
+	case DATA_TYPE_UINT64:
+		if (size != sizeof (uint64_t)) {
+			rv = EINVAL;
+			break;
+		}
+		rv = nvlist_add_uint64(nv, key, *(uint64_t *)value);
+		break;
+
+	case DATA_TYPE_STRING:
+		rv = nvlist_add_string(nv, key, value);
+		break;
+
+	case DATA_TYPE_BYTE_ARRAY:
+		rv = nvlist_add_byte_array(nv, key, value, size);
+		break;
+
+	case DATA_TYPE_INT16_ARRAY:
+		rv = nvlist_add_int16_array(nv, key, value, size);
+		break;
+
+	case DATA_TYPE_UINT16_ARRAY:
+		rv = nvlist_add_uint16_array(nv, key, value, size);
+		break;
+
+	case DATA_TYPE_INT32_ARRAY:
+		rv = nvlist_add_int32_array(nv, key, value, size);
+		break;
+
+	case DATA_TYPE_UINT32_ARRAY:
+		rv = nvlist_add_uint32_array(nv, key, value, size);
+		break;
+
+	case DATA_TYPE_INT64_ARRAY:
+		rv = nvlist_add_int64_array(nv, key, value, size);
+		break;
+
+	case DATA_TYPE_UINT64_ARRAY:
+		rv = nvlist_add_uint64_array(nv, key, value, size);
+		break;
+
+	case DATA_TYPE_STRING_ARRAY:
+		rv = nvlist_add_string_array(nv, key, value, size);
+		break;
+
+	case DATA_TYPE_NVLIST:
+		rv = nvlist_add_nvlist(nv, key, (nvlist_t *)value);
+		break;
+
+	case DATA_TYPE_NVLIST_ARRAY:
+		rv = nvlist_add_nvlist_array(nv, key, value, size);
+		break;
+
+	case DATA_TYPE_BOOLEAN_VALUE:
+		if (size != sizeof (boolean_t)) {
+			rv = EINVAL;
+			break;
+		}
+		rv = nvlist_add_boolean_value(nv, key, *(boolean_t *)value);
+		break;
+
+	case DATA_TYPE_INT8:
+		if (size != sizeof (int8_t)) {
+			rv = EINVAL;
+			break;
+		}
+		rv = nvlist_add_int8(nv, key, *(int8_t *)value);
+		break;
+
+	case DATA_TYPE_UINT8:
+		if (size != sizeof (uint8_t)) {
+			rv = EINVAL;
+			break;
+		}
+		rv = nvlist_add_uint8(nv, key, *(uint8_t *)value);
+		break;
+
+	case DATA_TYPE_BOOLEAN_ARRAY:
+		rv = nvlist_add_boolean_array(nv, key, value, size);
+		break;
+
+	case DATA_TYPE_INT8_ARRAY:
+		rv = nvlist_add_int8_array(nv, key, value, size);
+		break;
+
+	case DATA_TYPE_UINT8_ARRAY:
+		rv = nvlist_add_uint8_array(nv, key, value, size);
+		break;
+
+	default:
+		return (ENOTSUP);
+	}
+
+	return (rv);
+}
+
+int
+lzbe_remove_pair(void *ptr, const char *key)
+{
+
+	return (nvlist_remove_all(ptr, key));
+}

lib/libzfsbootenv/lzbe_util.c

@@ -0,0 +1,39 @@
+/*
+ * This file and its contents are supplied under the terms of the
+ * Common Development and Distribution License ("CDDL"), version 1.0.
+ * You may only use this file in accordance with the terms of version
+ * 1.0 of the CDDL.
+ *
+ * A full copy of the text of the CDDL should have accompanied this
+ * source.  A copy of the CDDL is also available via the Internet at
+ * http://www.illumos.org/license/CDDL.
+ */
+/*
+ * Copyright 2020 Toomas Soome <tsoome@me.com>
+ */
+
+#include <sys/types.h>
+#include <string.h>
+#include <libzfs.h>
+#include <libzfsbootenv.h>
+
+/*
+ * Output bootenv information.
+ */
+int
+lzbe_bootenv_print(const char *pool, const char *nvlist, FILE *of)
+{
+	nvlist_t *nv;
+	int rv = -1;
+
+	if (pool == NULL || *pool == '\0' || of == NULL)
+		return (rv);
+
+	rv = lzbe_nvlist_get(pool, nvlist, (void **)&nv);
+	if (rv == 0) {
+		nvlist_print(of, nv);
+		nvlist_free(nv);
+	}
+
+	return (rv);
+}

lib/libzpool/Makefile.am

@@ -7,6 +7,13 @@ VPATH = \
 	$(top_srcdir)/module/os/linux/zfs \
 	$(top_srcdir)/lib/libzpool
 
+if BUILD_FREEBSD
+DEFAULT_INCLUDES += -I$(top_srcdir)/include/os/freebsd/zfs
+endif
+if BUILD_LINUX
+DEFAULT_INCLUDES += -I$(top_srcdir)/include/os/linux/zfs
+endif
+
 # Unconditionally enable debugging for libzpool
 AM_CPPFLAGS += -DDEBUG -UNDEBUG -DZFS_DEBUG
 

module/zfs/vdev_label.c

@@ -149,6 +149,8 @@
 #include <sys/dsl_scan.h>
 #include <sys/abd.h>
 #include <sys/fs/zfs.h>
+#include <sys/byteorder.h>
+#include <sys/zfs_bootenv.h>
 
 /*
  * Basic routines to read and write from a vdev label.
@@ -1233,13 +1235,9 @@ vdev_label_read_bootenv_impl(zio_t *zio, vdev_t *vd, int flags)
 	 * bootloader should have rewritten them all to be the same on boot,
 	 * and any changes we made since boot have been the same across all
 	 * labels.
-	 *
-	 * While grub supports writing to all four labels, other bootloaders
-	 * don't, so we only use the first two labels to store boot
-	 * information.
 	 */
 	if (vd->vdev_ops->vdev_op_leaf && vdev_readable(vd)) {
-		for (int l = 0; l < VDEV_LABELS / 2; l++) {
+		for (int l = 0; l < VDEV_LABELS; l++) {
 			vdev_label_read(zio, vd, l,
 			    abd_alloc_linear(VDEV_PAD_SIZE, B_FALSE),
 			    offsetof(vdev_label_t, vl_be), VDEV_PAD_SIZE,
@@ -1249,14 +1247,15 @@ vdev_label_read_bootenv_impl(zio_t *zio, vdev_t *vd, int flags)
 }
 
 int
-vdev_label_read_bootenv(vdev_t *rvd, nvlist_t *command)
+vdev_label_read_bootenv(vdev_t *rvd, nvlist_t *bootenv)
 {
+	nvlist_t *config;
 	spa_t *spa = rvd->vdev_spa;
 	abd_t *abd = NULL;
 	int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL |
 	    ZIO_FLAG_SPECULATIVE | ZIO_FLAG_TRYHARD;
 
-	ASSERT(command);
+	ASSERT(bootenv);
 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
 
 	zio_t *zio = zio_root(spa, NULL, &abd, flags);
@@ -1264,39 +1263,81 @@ vdev_label_read_bootenv(vdev_t *rvd, nvlist_t *command)
 	int err = zio_wait(zio);
 
 	if (abd != NULL) {
+		char *buf;
 		vdev_boot_envblock_t *vbe = abd_to_buf(abd);
-		if (vbe->vbe_version != VB_RAW) {
-			abd_free(abd);
-			return (SET_ERROR(ENOTSUP));
+
+		vbe->vbe_version = ntohll(vbe->vbe_version);
+		switch (vbe->vbe_version) {
+		case VB_RAW:
+			/*
+			 * if we have textual data in vbe_bootenv, create nvlist
+			 * with key "envmap".
+			 */
+			fnvlist_add_uint64(bootenv, BOOTENV_VERSION, VB_RAW);
+			vbe->vbe_bootenv[sizeof (vbe->vbe_bootenv) - 1] = '\0';
+			fnvlist_add_string(bootenv, GRUB_ENVMAP,
+			    vbe->vbe_bootenv);
+			break;
+
+		case VB_NVLIST:
+			err = nvlist_unpack(vbe->vbe_bootenv,
+			    sizeof (vbe->vbe_bootenv), &config, 0);
+			if (err == 0) {
+				fnvlist_merge(bootenv, config);
+				nvlist_free(config);
+				break;
+			}
+			/* FALLTHROUGH */
+		default:
+			/* Check for FreeBSD zfs bootonce command string */
+			buf = abd_to_buf(abd);
+			if (*buf == '\0') {
+				fnvlist_add_uint64(bootenv, BOOTENV_VERSION,
+				    VB_NVLIST);
+				break;
+			}
+			fnvlist_add_string(bootenv, FREEBSD_BOOTONCE, buf);
 		}
-		vbe->vbe_bootenv[sizeof (vbe->vbe_bootenv) - 1] = '\0';
-		fnvlist_add_string(command, "envmap", vbe->vbe_bootenv);
-		/* abd was allocated in vdev_label_read_bootenv_impl() */
+
+		/*
+		 * abd was allocated in vdev_label_read_bootenv_impl()
+		 */
 		abd_free(abd);
-		/* If we managed to read any successfully, return success. */
+		/*
+		 * If we managed to read any successfully,
+		 * return success.
+		 */
 		return (0);
 	}
 	return (err);
 }
 
 int
-vdev_label_write_bootenv(vdev_t *vd, char *envmap)
+vdev_label_write_bootenv(vdev_t *vd, nvlist_t *env)
 {
 	zio_t *zio;
 	spa_t *spa = vd->vdev_spa;
 	vdev_boot_envblock_t *bootenv;
 	int flags = ZIO_FLAG_CONFIG_WRITER | ZIO_FLAG_CANFAIL;
-	int error = ENXIO;
+	int error;
+	size_t nvsize;
+	char *nvbuf;
 
-	if (strlen(envmap) >= sizeof (bootenv->vbe_bootenv)) {
+	error = nvlist_size(env, &nvsize, NV_ENCODE_XDR);
+	if (error != 0)
+		return (SET_ERROR(error));
+
+	if (nvsize >= sizeof (bootenv->vbe_bootenv)) {
 		return (SET_ERROR(E2BIG));
 	}
 
 	ASSERT(spa_config_held(spa, SCL_ALL, RW_WRITER) == SCL_ALL);
 
+	error = ENXIO;
 	for (int c = 0; c < vd->vdev_children; c++) {
-		int child_err = vdev_label_write_bootenv(vd->vdev_child[c],
-		    envmap);
+		int child_err;
+
+		child_err = vdev_label_write_bootenv(vd->vdev_child[c], env);
 		/*
 		 * As long as any of the disks managed to write all of their
 		 * labels successfully, return success.
@@ -1312,16 +1353,41 @@ vdev_label_write_bootenv(vdev_t *vd, char *envmap)
 	ASSERT3U(sizeof (*bootenv), ==, VDEV_PAD_SIZE);
 	abd_t *abd = abd_alloc_for_io(VDEV_PAD_SIZE, B_TRUE);
 	abd_zero(abd, VDEV_PAD_SIZE);
-	bootenv = abd_borrow_buf_copy(abd, VDEV_PAD_SIZE);
 
-	char *buf = bootenv->vbe_bootenv;
-	(void) strlcpy(buf, envmap, sizeof (bootenv->vbe_bootenv));
-	bootenv->vbe_version = VB_RAW;
-	abd_return_buf_copy(abd, bootenv, VDEV_PAD_SIZE);
+	bootenv = abd_borrow_buf_copy(abd, VDEV_PAD_SIZE);
+	nvbuf = bootenv->vbe_bootenv;
+	nvsize = sizeof (bootenv->vbe_bootenv);
+
+	bootenv->vbe_version = fnvlist_lookup_uint64(env, BOOTENV_VERSION);
+	switch (bootenv->vbe_version) {
+	case VB_RAW:
+		if (nvlist_lookup_string(env, GRUB_ENVMAP, &nvbuf) == 0) {
+			(void) strlcpy(bootenv->vbe_bootenv, nvbuf, nvsize);
+		}
+		error = 0;
+		break;
+
+	case VB_NVLIST:
+		error = nvlist_pack(env, &nvbuf, &nvsize, NV_ENCODE_XDR,
+		    KM_SLEEP);
+		break;
+
+	default:
+		error = EINVAL;
+		break;
+	}
+
+	if (error == 0) {
+		bootenv->vbe_version = htonll(bootenv->vbe_version);
+		abd_return_buf_copy(abd, bootenv, VDEV_PAD_SIZE);
+	} else {
+		abd_free(abd);
+		return (SET_ERROR(error));
+	}
 
 retry:
 	zio = zio_root(spa, NULL, NULL, flags);
-	for (int l = 0; l < VDEV_LABELS / 2; l++) {
+	for (int l = 0; l < VDEV_LABELS; l++) {
 		vdev_label_write(zio, vd, l, abd,
 		    offsetof(vdev_label_t, vl_be),
 		    VDEV_PAD_SIZE, NULL, NULL, flags);

module/zfs/zfs_ioctl.c

@@ -3511,30 +3511,29 @@ zfs_ioc_log_history(const char *unused, nvlist_t *innvl, nvlist_t *outnvl)
 /*
  * This ioctl is used to set the bootenv configuration on the current
  * pool. This configuration is stored in the second padding area of the label,
- * and it is used by the GRUB bootloader used on Linux to store the contents
- * of the grubenv file.  The file is stored as raw ASCII, and is protected by
- * an embedded checksum.  By default, GRUB will check if the boot filesystem
- * supports storing the environment data in a special location, and if so,
- * will invoke filesystem specific logic to retrieve it. This can be overridden
- * by a variable, should the user so desire.
+ * and it is used by the bootloader(s) to store the bootloader and/or system
+ * specific data.
+ * The data is stored as nvlist data stream, and is protected by
+ * an embedded checksum.
+ * The version can have two possible values:
+ * VB_RAW: nvlist should have key GRUB_ENVMAP, value DATA_TYPE_STRING.
+ * VB_NVLIST: nvlist with arbitrary <key, value> pairs.
  */
-/* ARGSUSED */
 static const zfs_ioc_key_t zfs_keys_set_bootenv[] = {
-	{"envmap",	DATA_TYPE_STRING,	0},
+	{"version",	DATA_TYPE_UINT64,	0},
+	{"<keys>",	DATA_TYPE_ANY, ZK_OPTIONAL | ZK_WILDCARDLIST},
 };
 
 static int
 zfs_ioc_set_bootenv(const char *name, nvlist_t *innvl, nvlist_t *outnvl)
 {
-	char *envmap;
 	int error;
 	spa_t *spa;
 
-	envmap = fnvlist_lookup_string(innvl, "envmap");
 	if ((error = spa_open(name, &spa, FTAG)) != 0)
 		return (error);
 	spa_vdev_state_enter(spa, SCL_ALL);
-	error = vdev_label_write_bootenv(spa->spa_root_vdev, envmap);
+	error = vdev_label_write_bootenv(spa->spa_root_vdev, innvl);
 	(void) spa_vdev_state_exit(spa, NULL, 0);
 	spa_close(spa, FTAG);
 	return (error);
@@ -3544,7 +3543,6 @@ static const zfs_ioc_key_t zfs_keys_get_bootenv[] = {
 	/* no nvl keys */
 };
 
-/* ARGSUSED */
 static int
 zfs_ioc_get_bootenv(const char *name, nvlist_t *innvl, nvlist_t *outnvl)
 {

rpm/generic/zfs.spec.in

@@ -491,6 +491,7 @@ systemctl --system daemon-reload >/dev/null || true
 
 %files -n libzfs2-devel
 %{_pkgconfigdir}/libzfs.pc
+%{_pkgconfigdir}/libzfsbootenv.pc
 %{_pkgconfigdir}/libzfs_core.pc
 %{_libdir}/*.so
 %{_includedir}/*

tests/zfs-tests/cmd/libzfs_input_check/Makefile.am

@@ -4,6 +4,13 @@ pkgexecdir = $(datadir)/@PACKAGE@/zfs-tests/bin
 
 pkgexec_PROGRAMS = libzfs_input_check
 
+if BUILD_FREEBSD
+DEFAULT_INCLUDES += -I$(top_srcdir)/include/os/freebsd/zfs
+endif
+if BUILD_LINUX
+DEFAULT_INCLUDES += -I$(top_srcdir)/include/os/linux/zfs
+endif
+
 libzfs_input_check_SOURCES = libzfs_input_check.c
 libzfs_input_check_LDADD = \
 	$(abs_top_builddir)/lib/libzfs_core/libzfs_core.la \

tests/zfs-tests/cmd/libzfs_input_check/libzfs_input_check.c

@@ -25,7 +25,9 @@
 #include <libzutil.h>
 
 #include <sys/nvpair.h>
+#include <sys/vdev_impl.h>
 #include <sys/zfs_ioctl.h>
+#include <sys/zfs_bootenv.h>
 
 /*
  * Test the nvpair inputs for the non-legacy zfs ioctl commands.
@@ -762,9 +764,10 @@ test_set_bootenv(const char *pool)
 {
 	nvlist_t *required = fnvlist_alloc();
 
-	fnvlist_add_string(required, "envmap", "test");
+	fnvlist_add_uint64(required, "version", VB_RAW);
+	fnvlist_add_string(required, GRUB_ENVMAP, "test");
 
-	IOC_INPUT_TEST(ZFS_IOC_SET_BOOTENV, pool, required, NULL, 0);
+	IOC_INPUT_TEST_WILD(ZFS_IOC_SET_BOOTENV, pool, required, NULL, 0);
 
 	nvlist_free(required);
 }