Linux 5.10 compat: use iov_iter in uio structure

As of the 5.10 kernel the generic splice compatibility code has been
removed.  All filesystems are now responsible for registering a
->splice_read and ->splice_write callback to support this operation.

The good news is the VFS provided generic_file_splice_read() and
iter_file_splice_write() callbacks can be used provided the ->iter_read
and ->iter_write callback support pipes.  However, this is currently
not the case and only iovecs and bvecs (not pipes) are ever attached
to the uio structure.

This commit changes that by allowing full iov_iter structures to be
attached to uios.  Ever since the 4.9 kernel the iov_iter structure
has supported iovecs, kvecs, bvevs, and pipes so it's desirable to
pass the entire thing when possible.  In conjunction with this the
uio helper functions (i.e uiomove(), uiocopy(), etc) have been
updated to understand the new UIO_ITER type.

Note that using the kernel provided uio_iter interfaces allowed the
existing Linux specific uio handling code to be simplified.  When
there's no longer a need to support kernel's older than 4.9, then
it will be possible to remove the iovec and bvec members from the
uio structure and always use a uio_iter.  Until then we need to
maintain all of the existing types for older kernels.

Some additional refactoring and cleanup was included in this change:

- Added checks to configure to detect available iov_iter interfaces.
  Some are available all the way back to the 3.10 kernel and are used
  when available.  In particular, uio_prefaultpages() now always uses
  iov_iter_fault_in_readable() which is available for all supported
  kernels.

- The unused UIO_USERISPACE type has been removed.  It is no longer
  needed now that the uio_seg enum is platform specific.

- Moved zfs_uio.c from the zcommon.ko module to the Linux specific
  platform code for the zfs.ko module.  This gets it out of libzfs
  where it was never needed and keeps this Linux specific code out
  of the common sources.

- Removed unnecessary O_APPEND handling from zfs_iter_write(), this
  is redundant and O_APPEND is already handled in zfs_write();

NOTE: Cleanly applying this kernel compatibility change required
applying the following commits.  This makes the change larger than
it absolutely needs to be, but the resulting code matches what's in
the branch branch.  This is both more tested and makes it easier to
apply any future backports in this area.

7cf4cd824 Remove incorrect assertion
783be694f Reduce confusion in zfs_write
af5626ac2 Return EFAULT at the end of zfs_write() when set
cc1f85be8 Simplify offset and length limit in zfs_write
9585538d0 Const some unchanging variables in zfs_write
86e74dc16 Remove redundant oid parameter to update_pages
b3d723fb0 Factor uid, gid, and projid out of loop in zfs_write
3d40b6554 Share zfs_fsync, zfs_read, zfs_write, et al between Linux and FreeBSD

Reviewed-by: Colin Ian King <colin.king@canonical.com>
Reviewed-by: Tony Hutter <hutter2@llnl.gov>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #11351

module/os/linux/zfs/Makefile.in

@@ -23,6 +23,7 @@ $(MODULE)-objs += ../os/linux/zfs/zfs_dir.o
 $(MODULE)-objs += ../os/linux/zfs/zfs_file_os.o
 $(MODULE)-objs += ../os/linux/zfs/zfs_ioctl_os.o
 $(MODULE)-objs += ../os/linux/zfs/zfs_sysfs.o
+$(MODULE)-objs += ../os/linux/zfs/zfs_uio.o
 $(MODULE)-objs += ../os/linux/zfs/zfs_vfsops.o
 $(MODULE)-objs += ../os/linux/zfs/zfs_vnops_os.o
 $(MODULE)-objs += ../os/linux/zfs/zfs_znode.o

module/os/linux/zfs/zfs_uio.c

@@ -0,0 +1,296 @@
+/*
+ * CDDL HEADER START
+ *
+ * The contents of this file are subject to the terms of the
+ * Common Development and Distribution License (the "License").
+ * You may not use this file except in compliance with the License.
+ *
+ * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
+ * or http://www.opensolaris.org/os/licensing.
+ * See the License for the specific language governing permissions
+ * and limitations under the License.
+ *
+ * When distributing Covered Code, include this CDDL HEADER in each
+ * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
+ * If applicable, add the following below this CDDL HEADER, with the
+ * fields enclosed by brackets "[]" replaced with your own identifying
+ * information: Portions Copyright [yyyy] [name of copyright owner]
+ *
+ * CDDL HEADER END
+ */
+/*
+ * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
+ * Use is subject to license terms.
+ */
+
+/*	Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T	*/
+/*	  All Rights Reserved	*/
+
+/*
+ * University Copyright- Copyright (c) 1982, 1986, 1988
+ * The Regents of the University of California
+ * All Rights Reserved
+ *
+ * University Acknowledgment- Portions of this document are derived from
+ * software developed by the University of California, Berkeley, and its
+ * contributors.
+ */
+/*
+ * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
+ */
+
+#ifdef _KERNEL
+
+#include <sys/types.h>
+#include <sys/uio_impl.h>
+#include <sys/sysmacros.h>
+#include <sys/strings.h>
+#include <linux/kmap_compat.h>
+#include <linux/uaccess.h>
+
+/*
+ * Move "n" bytes at byte address "p"; "rw" indicates the direction
+ * of the move, and the I/O parameters are provided in "uio", which is
+ * update to reflect the data which was moved.  Returns 0 on success or
+ * a non-zero errno on failure.
+ */
+static int
+uiomove_iov(void *p, size_t n, enum uio_rw rw, struct uio *uio)
+{
+	const struct iovec *iov = uio->uio_iov;
+	size_t skip = uio->uio_skip;
+	ulong_t cnt;
+
+	while (n && uio->uio_resid) {
+		cnt = MIN(iov->iov_len - skip, n);
+		switch (uio->uio_segflg) {
+		case UIO_USERSPACE:
+			/*
+			 * p = kernel data pointer
+			 * iov->iov_base = user data pointer
+			 */
+			if (rw == UIO_READ) {
+				if (copy_to_user(iov->iov_base+skip, p, cnt))
+					return (EFAULT);
+			} else {
+				unsigned long b_left = 0;
+				if (uio->uio_fault_disable) {
+					if (!zfs_access_ok(VERIFY_READ,
+					    (iov->iov_base + skip), cnt)) {
+						return (EFAULT);
+					}
+					pagefault_disable();
+					b_left =
+					    __copy_from_user_inatomic(p,
+					    (iov->iov_base + skip), cnt);
+					pagefault_enable();
+				} else {
+					b_left =
+					    copy_from_user(p,
+					    (iov->iov_base + skip), cnt);
+				}
+				if (b_left > 0) {
+					unsigned long c_bytes =
+					    cnt - b_left;
+					uio->uio_skip += c_bytes;
+					ASSERT3U(uio->uio_skip, <,
+					    iov->iov_len);
+					uio->uio_resid -= c_bytes;
+					uio->uio_loffset += c_bytes;
+					return (EFAULT);
+				}
+			}
+			break;
+		case UIO_SYSSPACE:
+			if (rw == UIO_READ)
+				bcopy(p, iov->iov_base + skip, cnt);
+			else
+				bcopy(iov->iov_base + skip, p, cnt);
+			break;
+		default:
+			ASSERT(0);
+		}
+		skip += cnt;
+		if (skip == iov->iov_len) {
+			skip = 0;
+			uio->uio_iov = (++iov);
+			uio->uio_iovcnt--;
+		}
+		uio->uio_skip = skip;
+		uio->uio_resid -= cnt;
+		uio->uio_loffset += cnt;
+		p = (caddr_t)p + cnt;
+		n -= cnt;
+	}
+	return (0);
+}
+
+static int
+uiomove_bvec(void *p, size_t n, enum uio_rw rw, struct uio *uio)
+{
+	const struct bio_vec *bv = uio->uio_bvec;
+	size_t skip = uio->uio_skip;
+	ulong_t cnt;
+
+	while (n && uio->uio_resid) {
+		void *paddr;
+		cnt = MIN(bv->bv_len - skip, n);
+
+		paddr = zfs_kmap_atomic(bv->bv_page, KM_USER1);
+		if (rw == UIO_READ)
+			bcopy(p, paddr + bv->bv_offset + skip, cnt);
+		else
+			bcopy(paddr + bv->bv_offset + skip, p, cnt);
+		zfs_kunmap_atomic(paddr, KM_USER1);
+
+		skip += cnt;
+		if (skip == bv->bv_len) {
+			skip = 0;
+			uio->uio_bvec = (++bv);
+			uio->uio_iovcnt--;
+		}
+		uio->uio_skip = skip;
+		uio->uio_resid -= cnt;
+		uio->uio_loffset += cnt;
+		p = (caddr_t)p + cnt;
+		n -= cnt;
+	}
+	return (0);
+}
+
+#if defined(HAVE_VFS_IOV_ITER)
+static int
+uiomove_iter(void *p, size_t n, enum uio_rw rw, struct uio *uio,
+    boolean_t revert)
+{
+	size_t cnt = MIN(n, uio->uio_resid);
+
+	if (uio->uio_skip)
+		iov_iter_advance(uio->uio_iter, uio->uio_skip);
+
+	if (rw == UIO_READ)
+		cnt = copy_to_iter(p, cnt, uio->uio_iter);
+	else
+		cnt = copy_from_iter(p, cnt, uio->uio_iter);
+
+	/*
+	 * When operating on a full pipe no bytes are processed.
+	 * In which case return EFAULT which is converted to EAGAIN
+	 * by the kernel's generic_file_splice_read() function.
+	 */
+	if (cnt == 0)
+		return (EFAULT);
+
+	/*
+	 * Revert advancing the uio_iter.  This is set by uiocopy()
+	 * to avoid consuming the uio and its iov_iter structure.
+	 */
+	if (revert)
+		iov_iter_revert(uio->uio_iter, cnt);
+
+	uio->uio_resid -= cnt;
+	uio->uio_loffset += cnt;
+
+	return (0);
+}
+#endif
+
+int
+uiomove(void *p, size_t n, enum uio_rw rw, struct uio *uio)
+{
+	if (uio->uio_segflg == UIO_BVEC)
+		return (uiomove_bvec(p, n, rw, uio));
+#if defined(HAVE_VFS_IOV_ITER)
+	else if (uio->uio_segflg == UIO_ITER)
+		return (uiomove_iter(p, n, rw, uio, B_FALSE));
+#endif
+	else
+		return (uiomove_iov(p, n, rw, uio));
+}
+EXPORT_SYMBOL(uiomove);
+
+int
+uio_prefaultpages(ssize_t n, struct uio *uio)
+{
+	struct iov_iter iter, *iterp = NULL;
+
+#if defined(HAVE_IOV_ITER_FAULT_IN_READABLE)
+	if (uio->uio_segflg == UIO_USERSPACE) {
+		iterp = &iter;
+		iov_iter_init_compat(iterp, READ, uio->uio_iov,
+		    uio->uio_iovcnt, uio->uio_resid);
+#if defined(HAVE_VFS_IOV_ITER)
+	} else if (uio->uio_segflg == UIO_ITER) {
+		iterp = uio->uio_iter;
+#endif
+	}
+
+	if (iterp && iov_iter_fault_in_readable(iterp, n))
+		return (EFAULT);
+#endif
+	return (0);
+}
+EXPORT_SYMBOL(uio_prefaultpages);
+
+/*
+ * The same as uiomove() but doesn't modify uio structure.
+ * return in cbytes how many bytes were copied.
+ */
+int
+uiocopy(void *p, size_t n, enum uio_rw rw, struct uio *uio, size_t *cbytes)
+{
+	struct uio uio_copy;
+	int ret;
+
+	bcopy(uio, &uio_copy, sizeof (struct uio));
+
+	if (uio->uio_segflg == UIO_BVEC)
+		ret = uiomove_bvec(p, n, rw, &uio_copy);
+#if defined(HAVE_VFS_IOV_ITER)
+	else if (uio->uio_segflg == UIO_ITER)
+		ret = uiomove_iter(p, n, rw, &uio_copy, B_TRUE);
+#endif
+	else
+		ret = uiomove_iov(p, n, rw, &uio_copy);
+
+	*cbytes = uio->uio_resid - uio_copy.uio_resid;
+
+	return (ret);
+}
+EXPORT_SYMBOL(uiocopy);
+
+/*
+ * Drop the next n chars out of *uio.
+ */
+void
+uioskip(uio_t *uio, size_t n)
+{
+	if (n > uio->uio_resid)
+		return;
+
+	if (uio->uio_segflg == UIO_BVEC) {
+		uio->uio_skip += n;
+		while (uio->uio_iovcnt &&
+		    uio->uio_skip >= uio->uio_bvec->bv_len) {
+			uio->uio_skip -= uio->uio_bvec->bv_len;
+			uio->uio_bvec++;
+			uio->uio_iovcnt--;
+		}
+#if defined(HAVE_VFS_IOV_ITER)
+	} else if (uio->uio_segflg == UIO_ITER) {
+		iov_iter_advance(uio->uio_iter, n);
+#endif
+	} else {
+		uio->uio_skip += n;
+		while (uio->uio_iovcnt &&
+		    uio->uio_skip >= uio->uio_iov->iov_len) {
+			uio->uio_skip -= uio->uio_iov->iov_len;
+			uio->uio_iov++;
+			uio->uio_iovcnt--;
+		}
+	}
+	uio->uio_loffset += n;
+	uio->uio_resid -= n;
+}
+EXPORT_SYMBOL(uioskip);
+#endif /* _KERNEL */

module/os/linux/zfs/zfs_vnops_os.c

@@ -427,28 +427,37 @@ unsigned long zfs_delete_blocks = DMU_MAX_DELETEBLKCNT;
  *	OUT:	resid	- remaining bytes to write
  *
  *	RETURN:	0 if success
- *		positive error code if failure
+ *		positive error code if failure.  EIO is	returned
+ *		for a short write when residp isn't provided.
  *
  * Timestamps:
  *	zp - ctime|mtime updated if byte count > 0
  */
 int
 zfs_write_simple(znode_t *zp, const void *data, size_t len,
-    loff_t pos, size_t *resid)
+    loff_t pos, size_t *residp)
 {
-	ssize_t written;
-	int error = 0;
+	fstrans_cookie_t cookie;
+	int error;
 
-	written = zpl_write_common(ZTOI(zp), data, len, &pos,
-	    UIO_SYSSPACE, 0, kcred);
-	if (written < 0) {
-		error = -written;
-	} else if (resid == NULL) {
-		if (written < len)
-			error = SET_ERROR(EIO); /* short write */
-	} else {
-		*resid = len - written;
+	struct iovec iov;
+	iov.iov_base = (void *)data;
+	iov.iov_len = len;
+
+	uio_t uio;
+	uio_iovec_init(&uio, &iov, 1, pos, UIO_SYSSPACE, len, 0);
+
+	cookie = spl_fstrans_mark();
+	error = zfs_write(zp, &uio, 0, kcred);
+	spl_fstrans_unmark(cookie);
+
+	if (error == 0) {
+		if (residp != NULL)
+			*residp = uio_resid(&uio);
+		else if (uio_resid(&uio) != 0)
+			error = SET_ERROR(EIO);
 	}
+
 	return (error);
 }
 

module/os/linux/zfs/zpl_file.c

@@ -212,242 +212,224 @@ zfs_io_flags(struct kiocb *kiocb)
 	return (flags);
 }
 
-static ssize_t
-zpl_read_common_iovec(struct inode *ip, const struct iovec *iovp, size_t count,
-    unsigned long nr_segs, loff_t *ppos, uio_seg_t segment, int flags,
-    cred_t *cr, size_t skip)
+/*
+ * If relatime is enabled, call file_accessed() if zfs_relatime_need_update()
+ * is true.  This is needed since datasets with inherited "relatime" property
+ * aren't necessarily mounted with the MNT_RELATIME flag (e.g. after
+ * `zfs set relatime=...`), which is what relatime test in VFS by
+ * relatime_need_update() is based on.
+ */
+static inline void
+zpl_file_accessed(struct file *filp)
 {
-	ssize_t read;
-	uio_t uio = { { 0 }, 0 };
-	int error;
-	fstrans_cookie_t cookie;
-
-	uio.uio_iov = iovp;
-	uio.uio_iovcnt = nr_segs;
-	uio.uio_loffset = *ppos;
-	uio.uio_segflg = segment;
-	uio.uio_resid = count;
-	uio.uio_skip = skip;
-
-	cookie = spl_fstrans_mark();
-	error = -zfs_read(ITOZ(ip), &uio, flags, cr);
-	spl_fstrans_unmark(cookie);
-	if (error < 0)
-		return (error);
-
-	read = count - uio.uio_resid;
-	*ppos += read;
-
-	return (read);
-}
-
-inline ssize_t
-zpl_read_common(struct inode *ip, const char *buf, size_t len, loff_t *ppos,
-    uio_seg_t segment, int flags, cred_t *cr)
-{
-	struct iovec iov;
-
-	iov.iov_base = (void *)buf;
-	iov.iov_len = len;
-
-	return (zpl_read_common_iovec(ip, &iov, len, 1, ppos, segment,
-	    flags, cr, 0));
-}
-
-static ssize_t
-zpl_iter_read_common(struct kiocb *kiocb, const struct iovec *iovp,
-    unsigned long nr_segs, size_t count, uio_seg_t seg, size_t skip)
-{
-	cred_t *cr = CRED();
-	struct file *filp = kiocb->ki_filp;
 	struct inode *ip = filp->f_mapping->host;
-	zfsvfs_t *zfsvfs = ZTOZSB(ITOZ(ip));
-	ssize_t read;
-	unsigned int f_flags = filp->f_flags;
 
-	f_flags |= zfs_io_flags(kiocb);
-	crhold(cr);
-	read = zpl_read_common_iovec(filp->f_mapping->host, iovp, count,
-	    nr_segs, &kiocb->ki_pos, seg, f_flags, cr, skip);
-	crfree(cr);
-
-	/*
-	 * If relatime is enabled, call file_accessed() only if
-	 * zfs_relatime_need_update() is true.  This is needed since datasets
-	 * with inherited "relatime" property aren't necessarily mounted with
-	 * MNT_RELATIME flag (e.g. after `zfs set relatime=...`), which is what
-	 * relatime test in VFS by relatime_need_update() is based on.
-	 */
-	if (!IS_NOATIME(ip) && zfsvfs->z_relatime) {
+	if (!IS_NOATIME(ip) && ITOZSB(ip)->z_relatime) {
 		if (zfs_relatime_need_update(ip))
 			file_accessed(filp);
 	} else {
 		file_accessed(filp);
 	}
+}
+
+#if defined(HAVE_VFS_RW_ITERATE)
+
+/*
+ * When HAVE_VFS_IOV_ITER is defined the iov_iter structure supports
+ * iovecs, kvevs, bvecs and pipes, plus all the required interfaces to
+ * manipulate the iov_iter are available.  In which case the full iov_iter
+ * can be attached to the uio and correctly handled in the lower layers.
+ * Otherwise, for older kernels extract the iovec and pass it instead.
+ */
+static void
+zpl_uio_init(uio_t *uio, struct kiocb *kiocb, struct iov_iter *to,
+    loff_t pos, ssize_t count, size_t skip)
+{
+#if defined(HAVE_VFS_IOV_ITER)
+	uio_iov_iter_init(uio, to, pos, count, skip);
+#else
+	uio_iovec_init(uio, to->iov, to->nr_segs, pos,
+	    to->type & ITER_KVEC ? UIO_SYSSPACE : UIO_USERSPACE,
+	    count, skip);
+#endif
+}
+
+static ssize_t
+zpl_iter_read(struct kiocb *kiocb, struct iov_iter *to)
+{
+	cred_t *cr = CRED();
+	fstrans_cookie_t cookie;
+	struct file *filp = kiocb->ki_filp;
+	ssize_t count = iov_iter_count(to);
+	uio_t uio;
+
+	zpl_uio_init(&uio, kiocb, to, kiocb->ki_pos, count, 0);
+
+	crhold(cr);
+	cookie = spl_fstrans_mark();
+
+	int error = -zfs_read(ITOZ(filp->f_mapping->host), &uio,
+	    filp->f_flags | zfs_io_flags(kiocb), cr);
+
+	spl_fstrans_unmark(cookie);
+	crfree(cr);
+
+	if (error < 0)
+		return (error);
+
+	ssize_t read = count - uio.uio_resid;
+	kiocb->ki_pos += read;
+
+	zpl_file_accessed(filp);
+
+	if (read > 0)
+		iov_iter_advance(to, read);
 
 	return (read);
 }
 
-#if defined(HAVE_VFS_RW_ITERATE)
-static ssize_t
-zpl_iter_read(struct kiocb *kiocb, struct iov_iter *to)
+static inline ssize_t
+zpl_generic_write_checks(struct kiocb *kiocb, struct iov_iter *from,
+    size_t *countp)
 {
-	ssize_t ret;
-	uio_seg_t seg = UIO_USERSPACE;
-	if (to->type & ITER_KVEC)
-		seg = UIO_SYSSPACE;
-	if (to->type & ITER_BVEC)
-		seg = UIO_BVEC;
-	ret = zpl_iter_read_common(kiocb, to->iov, to->nr_segs,
-	    iov_iter_count(to), seg, to->iov_offset);
-	if (ret > 0)
-		iov_iter_advance(to, ret);
-	return (ret);
-}
-#else
-static ssize_t
-zpl_aio_read(struct kiocb *kiocb, const struct iovec *iovp,
-    unsigned long nr_segs, loff_t pos)
-{
-	ssize_t ret;
-	size_t count;
-
-	ret = generic_segment_checks(iovp, &nr_segs, &count, VERIFY_WRITE);
-	if (ret)
+#ifdef HAVE_GENERIC_WRITE_CHECKS_KIOCB
+	ssize_t ret = generic_write_checks(kiocb, from);
+	if (ret <= 0)
 		return (ret);
 
-	return (zpl_iter_read_common(kiocb, iovp, nr_segs, count,
-	    UIO_USERSPACE, 0));
-}
-#endif /* HAVE_VFS_RW_ITERATE */
+	*countp = ret;
+#else
+	struct file *file = kiocb->ki_filp;
+	struct address_space *mapping = file->f_mapping;
+	struct inode *ip = mapping->host;
+	int isblk = S_ISBLK(ip->i_mode);
 
-static ssize_t
-zpl_write_common_iovec(struct inode *ip, const struct iovec *iovp, size_t count,
-    unsigned long nr_segs, loff_t *ppos, uio_seg_t segment, int flags,
-    cred_t *cr, size_t skip)
-{
-	ssize_t wrote;
-	uio_t uio = { { 0 }, 0 };
-	int error;
-	fstrans_cookie_t cookie;
+	*countp = iov_iter_count(from);
+	ssize_t ret = generic_write_checks(file, &kiocb->ki_pos, countp, isblk);
+	if (ret)
+		return (ret);
+#endif
 
-	if (flags & O_APPEND)
-		*ppos = i_size_read(ip);
-
-	uio.uio_iov = iovp;
-	uio.uio_iovcnt = nr_segs;
-	uio.uio_loffset = *ppos;
-	uio.uio_segflg = segment;
-	uio.uio_resid = count;
-	uio.uio_skip = skip;
-
-	cookie = spl_fstrans_mark();
-	error = -zfs_write(ITOZ(ip), &uio, flags, cr);
-	spl_fstrans_unmark(cookie);
-	if (error < 0)
-		return (error);
-
-	wrote = count - uio.uio_resid;
-	*ppos += wrote;
-
-	return (wrote);
+	return (0);
 }
 
-inline ssize_t
-zpl_write_common(struct inode *ip, const char *buf, size_t len, loff_t *ppos,
-    uio_seg_t segment, int flags, cred_t *cr)
-{
-	struct iovec iov;
-
-	iov.iov_base = (void *)buf;
-	iov.iov_len = len;
-
-	return (zpl_write_common_iovec(ip, &iov, len, 1, ppos, segment,
-	    flags, cr, 0));
-}
-
-static ssize_t
-zpl_iter_write_common(struct kiocb *kiocb, const struct iovec *iovp,
-    unsigned long nr_segs, size_t count, uio_seg_t seg, size_t skip)
-{
-	cred_t *cr = CRED();
-	struct file *filp = kiocb->ki_filp;
-	ssize_t wrote;
-	unsigned int f_flags = filp->f_flags;
-
-	f_flags |= zfs_io_flags(kiocb);
-	crhold(cr);
-	wrote = zpl_write_common_iovec(filp->f_mapping->host, iovp, count,
-	    nr_segs, &kiocb->ki_pos, seg, f_flags, cr, skip);
-	crfree(cr);
-
-	return (wrote);
-}
-
-#if defined(HAVE_VFS_RW_ITERATE)
 static ssize_t
 zpl_iter_write(struct kiocb *kiocb, struct iov_iter *from)
 {
+	cred_t *cr = CRED();
+	fstrans_cookie_t cookie;
+	struct file *filp = kiocb->ki_filp;
+	struct inode *ip = filp->f_mapping->host;
+	uio_t uio;
 	size_t count;
 	ssize_t ret;
-	uio_seg_t seg = UIO_USERSPACE;
 
-#ifndef HAVE_GENERIC_WRITE_CHECKS_KIOCB
-	struct file *file = kiocb->ki_filp;
-	struct address_space *mapping = file->f_mapping;
-	struct inode *ip = mapping->host;
-	int isblk = S_ISBLK(ip->i_mode);
-
-	count = iov_iter_count(from);
-	ret = generic_write_checks(file, &kiocb->ki_pos, &count, isblk);
+	ret = zpl_generic_write_checks(kiocb, from, &count);
 	if (ret)
 		return (ret);
-#else
-	/*
-	 * XXX - ideally this check should be in the same lock region with
-	 * write operations, so that there's no TOCTTOU race when doing
-	 * append and someone else grow the file.
-	 */
-	ret = generic_write_checks(kiocb, from);
-	if (ret <= 0)
-		return (ret);
-	count = ret;
-#endif
 
-	if (from->type & ITER_KVEC)
-		seg = UIO_SYSSPACE;
-	if (from->type & ITER_BVEC)
-		seg = UIO_BVEC;
+	zpl_uio_init(&uio, kiocb, from, kiocb->ki_pos, count, from->iov_offset);
 
-	ret = zpl_iter_write_common(kiocb, from->iov, from->nr_segs,
-	    count, seg, from->iov_offset);
-	if (ret > 0)
-		iov_iter_advance(from, ret);
+	crhold(cr);
+	cookie = spl_fstrans_mark();
 
-	return (ret);
+	int error = -zfs_write(ITOZ(ip), &uio,
+	    filp->f_flags | zfs_io_flags(kiocb), cr);
+
+	spl_fstrans_unmark(cookie);
+	crfree(cr);
+
+	if (error < 0)
+		return (error);
+
+	ssize_t wrote = count - uio.uio_resid;
+	kiocb->ki_pos += wrote;
+
+	if (wrote > 0)
+		iov_iter_advance(from, wrote);
+
+	return (wrote);
 }
-#else
+
+#else /* !HAVE_VFS_RW_ITERATE */
+
 static ssize_t
-zpl_aio_write(struct kiocb *kiocb, const struct iovec *iovp,
+zpl_aio_read(struct kiocb *kiocb, const struct iovec *iov,
     unsigned long nr_segs, loff_t pos)
 {
-	struct file *file = kiocb->ki_filp;
-	struct address_space *mapping = file->f_mapping;
-	struct inode *ip = mapping->host;
-	int isblk = S_ISBLK(ip->i_mode);
+	cred_t *cr = CRED();
+	fstrans_cookie_t cookie;
+	struct file *filp = kiocb->ki_filp;
 	size_t count;
 	ssize_t ret;
 
-	ret = generic_segment_checks(iovp, &nr_segs, &count, VERIFY_READ);
+	ret = generic_segment_checks(iov, &nr_segs, &count, VERIFY_WRITE);
 	if (ret)
 		return (ret);
 
-	ret = generic_write_checks(file, &pos, &count, isblk);
+	uio_t uio;
+	uio_iovec_init(&uio, iov, nr_segs, kiocb->ki_pos, UIO_USERSPACE,
+	    count, 0);
+
+	crhold(cr);
+	cookie = spl_fstrans_mark();
+
+	int error = -zfs_read(ITOZ(filp->f_mapping->host), &uio,
+	    filp->f_flags | zfs_io_flags(kiocb), cr);
+
+	spl_fstrans_unmark(cookie);
+	crfree(cr);
+
+	if (error < 0)
+		return (error);
+
+	ssize_t read = count - uio.uio_resid;
+	kiocb->ki_pos += read;
+
+	zpl_file_accessed(filp);
+
+	return (read);
+}
+
+static ssize_t
+zpl_aio_write(struct kiocb *kiocb, const struct iovec *iov,
+    unsigned long nr_segs, loff_t pos)
+{
+	cred_t *cr = CRED();
+	fstrans_cookie_t cookie;
+	struct file *filp = kiocb->ki_filp;
+	struct inode *ip = filp->f_mapping->host;
+	size_t count;
+	ssize_t ret;
+
+	ret = generic_segment_checks(iov, &nr_segs, &count, VERIFY_READ);
 	if (ret)
 		return (ret);
 
-	return (zpl_iter_write_common(kiocb, iovp, nr_segs, count,
-	    UIO_USERSPACE, 0));
+	ret = generic_write_checks(filp, &pos, &count, S_ISBLK(ip->i_mode));
+	if (ret)
+		return (ret);
+
+	uio_t uio;
+	uio_iovec_init(&uio, iov, nr_segs, kiocb->ki_pos, UIO_USERSPACE,
+	    count, 0);
+
+	crhold(cr);
+	cookie = spl_fstrans_mark();
+
+	int error = -zfs_write(ITOZ(ip), &uio,
+	    filp->f_flags | zfs_io_flags(kiocb), cr);
+
+	spl_fstrans_unmark(cookie);
+	crfree(cr);
+
+	if (error < 0)
+		return (error);
+
+	ssize_t wrote = count - uio.uio_resid;
+	kiocb->ki_pos += wrote;
+
+	return (wrote);
 }
 #endif /* HAVE_VFS_RW_ITERATE */
 
@@ -488,13 +470,13 @@ zpl_direct_IO(int rw, struct kiocb *kiocb, struct iov_iter *iter, loff_t pos)
 
 #if defined(HAVE_VFS_DIRECT_IO_IOVEC)
 static ssize_t
-zpl_direct_IO(int rw, struct kiocb *kiocb, const struct iovec *iovp,
+zpl_direct_IO(int rw, struct kiocb *kiocb, const struct iovec *iov,
     loff_t pos, unsigned long nr_segs)
 {
 	if (rw == WRITE)
-		return (zpl_aio_write(kiocb, iovp, nr_segs, pos));
+		return (zpl_aio_write(kiocb, iov, nr_segs, pos));
 	else
-		return (zpl_aio_read(kiocb, iovp, nr_segs, pos));
+		return (zpl_aio_read(kiocb, iov, nr_segs, pos));
 }
 #else
 #error "Unknown direct IO interface"
@@ -601,10 +583,6 @@ zpl_mmap(struct file *filp, struct vm_area_struct *vma)
  * Populate a page with data for the Linux page cache.  This function is
  * only used to support mmap(2).  There will be an identical copy of the
  * data in the ARC which is kept up to date via .write() and .writepage().
- *
- * Current this function relies on zpl_read_common() and the O_DIRECT
- * flag to read in a page.  This works but the more correct way is to
- * update zfs_fillpage() to be Linux friendly and use that interface.
  */
 static int
 zpl_readpage(struct file *filp, struct page *pp)
@@ -1035,6 +1013,10 @@ const struct file_operations zpl_file_operations = {
 #endif
 	.read_iter	= zpl_iter_read,
 	.write_iter	= zpl_iter_write,
+#ifdef HAVE_VFS_IOV_ITER
+	.splice_read	= generic_file_splice_read,
+	.splice_write	= iter_file_splice_write,
+#endif
 #else
 	.read		= do_sync_read,
 	.write		= do_sync_write,

module/os/linux/zfs/zpl_inode.c

@@ -490,19 +490,17 @@ zpl_get_link_common(struct dentry *dentry, struct inode *ip, char **link)
 {
 	fstrans_cookie_t cookie;
 	cred_t *cr = CRED();
-	struct iovec iov;
-	uio_t uio = { { 0 }, 0 };
 	int error;
 
 	crhold(cr);
 	*link = NULL;
+
+	struct iovec iov;
 	iov.iov_len = MAXPATHLEN;
 	iov.iov_base = kmem_zalloc(MAXPATHLEN, KM_SLEEP);
 
-	uio.uio_iov = &iov;
-	uio.uio_iovcnt = 1;
-	uio.uio_segflg = UIO_SYSSPACE;
-	uio.uio_resid = (MAXPATHLEN - 1);
+	uio_t uio;
+	uio_iovec_init(&uio, &iov, 1, 0, UIO_SYSSPACE, MAXPATHLEN - 1, 0);
 
 	cookie = spl_fstrans_mark();
 	error = -zfs_readlink(ip, &uio, cr);

module/os/linux/zfs/zpl_xattr.c

@@ -274,10 +274,10 @@ static int
 zpl_xattr_get_dir(struct inode *ip, const char *name, void *value,
     size_t size, cred_t *cr)
 {
+	fstrans_cookie_t cookie;
 	struct inode *xip = NULL;
 	znode_t *dxzp = NULL;
 	znode_t *xzp = NULL;
-	loff_t pos = 0;
 	int error;
 
 	/* Lookup the xattr directory */
@@ -302,7 +302,19 @@ zpl_xattr_get_dir(struct inode *ip, const char *name, void *value,
 		goto out;
 	}
 
-	error = zpl_read_common(xip, value, size, &pos, UIO_SYSSPACE, 0, cr);
+	struct iovec iov;
+	iov.iov_base = (void *)value;
+	iov.iov_len = size;
+
+	uio_t uio;
+	uio_iovec_init(&uio, &iov, 1, 0, UIO_SYSSPACE, size, 0);
+
+	cookie = spl_fstrans_mark();
+	error = -zfs_read(ITOZ(xip), &uio, 0, cr);
+	spl_fstrans_unmark(cookie);
+
+	if (error == 0)
+		error = size - uio_resid(&uio);
 out:
 	if (xzp)
 		zrele(xzp);
@@ -441,7 +453,6 @@ zpl_xattr_set_dir(struct inode *ip, const char *name, const void *value,
 	znode_t *dxzp = NULL;
 	znode_t *xzp = NULL;
 	vattr_t *vap = NULL;
-	ssize_t wrote;
 	int lookup_flags, error;
 	const int xattr_mode = S_IFREG | 0644;
 	loff_t pos = 0;
@@ -496,13 +507,8 @@ zpl_xattr_set_dir(struct inode *ip, const char *name, const void *value,
 	if (error)
 		goto out;
 
-	wrote = zpl_write_common(ZTOI(xzp), value, size, &pos,
-	    UIO_SYSSPACE, 0, cr);
-	if (wrote < 0)
-		error = wrote;
-
+	error = -zfs_write_simple(xzp, value, size, pos, NULL);
 out:
-
 	if (error == 0) {
 		ip->i_ctime = current_time(ip);
 		zfs_mark_inode_dirty(ip);

module/os/linux/zfs/zvol_os.c

@@ -84,26 +84,15 @@ zvol_is_zvol_impl(const char *device)
 	return (B_FALSE);
 }
 
-static void
-uio_from_bio(uio_t *uio, struct bio *bio)
-{
-	uio->uio_bvec = &bio->bi_io_vec[BIO_BI_IDX(bio)];
-	uio->uio_iovcnt = bio->bi_vcnt - BIO_BI_IDX(bio);
-	uio->uio_loffset = BIO_BI_SECTOR(bio) << 9;
-	uio->uio_segflg = UIO_BVEC;
-	uio->uio_resid = BIO_BI_SIZE(bio);
-	uio->uio_skip = BIO_BI_SKIP(bio);
-}
-
 static void
 zvol_write(void *arg)
 {
-	int error = 0;
-
 	zv_request_t *zvr = arg;
 	struct bio *bio = zvr->bio;
-	uio_t uio = { { 0 }, 0 };
-	uio_from_bio(&uio, bio);
+	int error = 0;
+	uio_t uio;
+
+	uio_bvec_init(&uio, bio);
 
 	zvol_state_t *zv = zvr->zv;
 	ASSERT3P(zv, !=, NULL);
@@ -249,12 +238,12 @@ unlock:
 static void
 zvol_read(void *arg)
 {
-	int error = 0;
-
 	zv_request_t *zvr = arg;
 	struct bio *bio = zvr->bio;
-	uio_t uio = { { 0 }, 0 };
-	uio_from_bio(&uio, bio);
+	int error = 0;
+	uio_t uio;
+
+	uio_bvec_init(&uio, bio);
 
 	zvol_state_t *zv = zvr->zv;
 	ASSERT3P(zv, !=, NULL);