Make zc_nvlist_src_size limit tunable

We limit the size of nvlists passed to the kernel so a user cannot make
the kernel do an unreasonably large allocation.  On FreeBSD this limit
was 128 kiB, which turns out to be a bit too small when doing some
operations involving a large number of datasets or snapshots, for
example replication.

Make this limit tunable, with a platform-specific auto default.
Linux keeps its limit at KMALLOC_MAX_SIZE. FreeBSD uses 1/4 of the
system limit on user wired memory, which allows it to scale depending
on system configuration.

Reviewed-by: Matt Macy <mmacy@FreeBSD.org>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Ryan Moeller <freqlabs@FreeBSD.org>
Issue #6572 
Closes #10706

module/os/freebsd/zfs/zfs_ioctl_os.c

@@ -35,9 +35,14 @@ __FBSDID("$FreeBSD$");
 #include <sys/vdev_os.h>
 #include <sys/zfs_vfsops.h>
 #include <sys/zone.h>
+#include <vm/vm_pageout.h>
 
 #include <sys/zfs_ioctl_impl.h>
 
+#if __FreeBSD_version < 1201517
+#define	vm_page_max_user_wired	vm_page_max_wired
+#endif
+
 int
 zfs_vfs_ref(zfsvfs_t **zfvp)
 {
@@ -133,6 +138,14 @@ zfs_ioc_nextboot(const char *unused, nvlist_t *innvl, nvlist_t *outnvl)
 	return (error);
 }
 
+uint64_t
+zfs_max_nvlist_src_size_os(void)
+{
+	if (zfs_max_nvlist_src_size != 0)
+		return (zfs_max_nvlist_src_size);
+
+	return (ptob(vm_page_max_user_wired) / 4);
+}
 
 void
 zfs_ioctl_init_os(void)

module/os/linux/zfs/zfs_ioctl_os.c

@@ -203,6 +203,15 @@ out:
 
 }
 
+uint64_t
+zfs_max_nvlist_src_size_os(void)
+{
+	if (zfs_max_nvlist_src_size != 0)
+		return (zfs_max_nvlist_src_size);
+
+	return (KMALLOC_MAX_SIZE);
+}
+
 void
 zfs_ioctl_init_os(void)
 {