Make zc_nvlist_src_size limit tunable

We limit the size of nvlists passed to the kernel so a user cannot make
the kernel do an unreasonably large allocation.  On FreeBSD this limit
was 128 kiB, which turns out to be a bit too small when doing some
operations involving a large number of datasets or snapshots, for
example replication.

Make this limit tunable, with a platform-specific auto default.
Linux keeps its limit at KMALLOC_MAX_SIZE. FreeBSD uses 1/4 of the
system limit on user wired memory, which allows it to scale depending
on system configuration.

Reviewed-by: Matt Macy <mmacy@FreeBSD.org>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Ryan Moeller <freqlabs@FreeBSD.org>
Issue #6572 
Closes #10706

module/os/freebsd/zfs/zfs_ioctl_os.c

@@ -35,9 +35,14 @@ __FBSDID("$FreeBSD$");
 #include <sys/vdev_os.h>
 #include <sys/zfs_vfsops.h>
 #include <sys/zone.h>
+#include <vm/vm_pageout.h>
 
 #include <sys/zfs_ioctl_impl.h>
 
+#if __FreeBSD_version < 1201517
+#define	vm_page_max_user_wired	vm_page_max_wired
+#endif
+
 int
 zfs_vfs_ref(zfsvfs_t **zfvp)
 {
@@ -133,6 +138,14 @@ zfs_ioc_nextboot(const char *unused, nvlist_t *innvl, nvlist_t *outnvl)
 	return (error);
 }
 
+uint64_t
+zfs_max_nvlist_src_size_os(void)
+{
+	if (zfs_max_nvlist_src_size != 0)
+		return (zfs_max_nvlist_src_size);
+
+	return (ptob(vm_page_max_user_wired) / 4);
+}
 
 void
 zfs_ioctl_init_os(void)

module/os/linux/zfs/zfs_ioctl_os.c

@@ -203,6 +203,15 @@ out:
 
 }
 
+uint64_t
+zfs_max_nvlist_src_size_os(void)
+{
+	if (zfs_max_nvlist_src_size != 0)
+		return (zfs_max_nvlist_src_size);
+
+	return (KMALLOC_MAX_SIZE);
+}
+
 void
 zfs_ioctl_init_os(void)
 {

module/zfs/zfs_ioctl.c

@@ -225,8 +225,9 @@ zfsdev_state_t *zfsdev_state_list;
 /*
  * Limit maximum nvlist size.  We don't want users passing in insane values
  * for zc->zc_nvlist_src_size, since we will need to allocate that much memory.
+ * Defaults to 0=auto which is handled by platform code.
  */
-#define	MAX_NVLIST_SRC_SIZE	KMALLOC_MAX_SIZE
+unsigned long zfs_max_nvlist_src_size = 0;
 
 uint_t zfs_fsyncer_key;
 uint_t zfs_allow_log_key;
@@ -7341,6 +7342,7 @@ zfsdev_ioctl_common(uint_t vecnum, zfs_cmd_t *zc, int flag)
 	int error, cmd;
 	const zfs_ioc_vec_t *vec;
 	char *saved_poolname = NULL;
+	uint64_t max_nvlist_src_size;
 	size_t saved_poolname_len = 0;
 	nvlist_t *innvl = NULL;
 	fstrans_cookie_t cookie;
@@ -7360,7 +7362,8 @@ zfsdev_ioctl_common(uint_t vecnum, zfs_cmd_t *zc, int flag)
 		return (SET_ERROR(ZFS_ERR_IOC_CMD_UNAVAIL));
 
 	zc->zc_iflags = flag & FKIOCTL;
-	if (zc->zc_nvlist_src_size > MAX_NVLIST_SRC_SIZE) {
+	max_nvlist_src_size = zfs_max_nvlist_src_size_os();
+	if (zc->zc_nvlist_src_size > max_nvlist_src_size) {
 		/*
 		 * Make sure the user doesn't pass in an insane value for
 		 * zc_nvlist_src_size.  We have to check, since we will end
@@ -7593,3 +7596,8 @@ zfs_kmod_fini(void)
 	tsd_destroy(&rrw_tsd_key);
 	tsd_destroy(&zfs_allow_log_key);
 }
+
+/* BEGIN CSTYLED */
+ZFS_MODULE_PARAM(zfs, zfs_, max_nvlist_src_size, ULONG, ZMOD_RW,
+    "Maximum size in bytes allowed for src nvlist passed with ZFS ioctls");
+/* END CSTYLED */