Make zc_nvlist_src_size limit tunable

We limit the size of nvlists passed to the kernel so a user cannot make
the kernel do an unreasonably large allocation.  On FreeBSD this limit
was 128 kiB, which turns out to be a bit too small when doing some
operations involving a large number of datasets or snapshots, for
example replication.

Make this limit tunable, with a platform-specific auto default.
Linux keeps its limit at KMALLOC_MAX_SIZE. FreeBSD uses 1/4 of the
system limit on user wired memory, which allows it to scale depending
on system configuration.

Reviewed-by: Matt Macy <mmacy@FreeBSD.org>
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Ryan Moeller <freqlabs@FreeBSD.org>
Issue #6572 
Closes #10706

include/os/freebsd/spl/sys/ccompile.h

@@ -162,8 +162,6 @@ extern "C" {
 #define	O_RSYNC 0
 #define	O_DSYNC 0
 
-#define	KMALLOC_MAX_SIZE MAXPHYS
-
 #ifndef LOCORE
 #ifndef HAVE_RPC_TYPES
 typedef int bool_t;

include/sys/zfs_ioctl_impl.h

@@ -25,6 +25,7 @@
 
 extern kmutex_t zfsdev_state_lock;
 extern zfsdev_state_t *zfsdev_state_list;
+extern unsigned long zfs_max_nvlist_src_size;
 
 typedef int zfs_ioc_legacy_func_t(zfs_cmd_t *);
 typedef int zfs_ioc_func_t(const char *, nvlist_t *, nvlist_t *);
@@ -80,6 +81,7 @@ void zfs_ioctl_register(const char *, zfs_ioc_t, zfs_ioc_func_t *,
     zfs_secpolicy_func_t *, zfs_ioc_namecheck_t, zfs_ioc_poolcheck_t,
     boolean_t, boolean_t, const zfs_ioc_key_t *, size_t);
 
+uint64_t zfs_max_nvlist_src_size_os(void);
 void zfs_ioctl_init_os(void);
 
 boolean_t zfs_vfs_held(zfsvfs_t *);