diff --git a/zfs-patches/0005-Fix-race-in-dnode_check_slots_free.patch b/zfs-patches/0005-Fix-race-in-dnode_check_slots_free.patch new file mode 100644 index 0000000..9cebd00 --- /dev/null +++ b/zfs-patches/0005-Fix-race-in-dnode_check_slots_free.patch @@ -0,0 +1,256 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Tom Caputi +Date: Tue, 10 Apr 2018 14:15:05 -0400 +Subject: [PATCH] Fix race in dnode_check_slots_free() + +Currently, dnode_check_slots_free() works by checking dn->dn_type +in the dnode to determine if the dnode is reclaimable. However, +there is a small window of time between dnode_free_sync() in the +first call to dsl_dataset_sync() and when the useraccounting code +is run when the type is set DMU_OT_NONE, but the dnode is not yet +evictable, leading to crashes. This patch adds the ability for +dnodes to track which txg they were last dirtied in and adds a +check for this before performing the reclaim. + +This patch also corrects several instances when dn_dirty_link was +treated as a list_node_t when it is technically a multilist_node_t. + +Reviewed-by: Brian Behlendorf +Signed-off-by: Tom Caputi +Requires-spl: spl-0.7-release +Issue #7147 +Issue #7388 +Issue #7997 + +(cherry-picked from behlendorf/issue-7997 4764f6f3be90be073d2700653dff286371e52583) +Signed-off-by: Stoiko Ivanov +--- + include/sys/dmu_impl.h | 1 + + include/sys/dnode.h | 4 ++++ + module/zfs/dbuf.c | 3 +++ + module/zfs/dmu.c | 2 +- + module/zfs/dmu_objset.c | 15 +++++++++++++++ + module/zfs/dnode.c | 29 +++++++++++++++++++---------- + 6 files changed, 43 insertions(+), 11 deletions(-) + +diff --git a/include/sys/dmu_impl.h b/include/sys/dmu_impl.h +index 65e417e3..03a63077 100644 +--- a/include/sys/dmu_impl.h ++++ b/include/sys/dmu_impl.h +@@ -161,6 +161,7 @@ extern "C" { + * dn_allocated_txg + * dn_free_txg + * dn_assigned_txg ++ * dn_dirty_txg + * dd_assigned_tx + * dn_notxholds + * dn_dirtyctx +diff --git a/include/sys/dnode.h b/include/sys/dnode.h +index ea7defe1..2dd087b3 100644 +--- a/include/sys/dnode.h ++++ b/include/sys/dnode.h +@@ -260,6 +260,7 @@ struct dnode { + uint64_t dn_allocated_txg; + uint64_t dn_free_txg; + uint64_t dn_assigned_txg; ++ uint64_t dn_dirty_txg; /* txg dnode was last dirtied */ + kcondvar_t dn_notxholds; + enum dnode_dirtycontext dn_dirtyctx; + uint8_t *dn_dirtyctx_firstset; /* dbg: contents meaningless */ +@@ -362,6 +363,9 @@ void dnode_evict_dbufs(dnode_t *dn); + void dnode_evict_bonus(dnode_t *dn); + void dnode_free_interior_slots(dnode_t *dn); + ++#define DNODE_IS_DIRTY(_dn) \ ++ ((_dn)->dn_dirty_txg >= spa_syncing_txg((_dn)->dn_objset->os_spa)) ++ + #define DNODE_IS_CACHEABLE(_dn) \ + ((_dn)->dn_objset->os_primary_cache == ZFS_CACHE_ALL || \ + (DMU_OT_IS_METADATA((_dn)->dn_type) && \ +diff --git a/module/zfs/dbuf.c b/module/zfs/dbuf.c +index 4ee121f5..6edb39d6 100644 +--- a/module/zfs/dbuf.c ++++ b/module/zfs/dbuf.c +@@ -1606,6 +1606,9 @@ dbuf_dirty(dmu_buf_impl_t *db, dmu_tx_t *tx) + FTAG); + } + } ++ ++ if (tx->tx_txg > dn->dn_dirty_txg) ++ dn->dn_dirty_txg = tx->tx_txg; + mutex_exit(&dn->dn_mtx); + + if (db->db_blkid == DMU_SPILL_BLKID) +diff --git a/module/zfs/dmu.c b/module/zfs/dmu.c +index 6f09aa2f..a09ac4f9 100644 +--- a/module/zfs/dmu.c ++++ b/module/zfs/dmu.c +@@ -2044,7 +2044,7 @@ dmu_offset_next(objset_t *os, uint64_t object, boolean_t hole, uint64_t *off) + * Check if dnode is dirty + */ + for (i = 0; i < TXG_SIZE; i++) { +- if (list_link_active(&dn->dn_dirty_link[i])) { ++ if (multilist_link_active(&dn->dn_dirty_link[i])) { + clean = B_FALSE; + break; + } +diff --git a/module/zfs/dmu_objset.c b/module/zfs/dmu_objset.c +index 449ebedf..0bed2d3e 100644 +--- a/module/zfs/dmu_objset.c ++++ b/module/zfs/dmu_objset.c +@@ -1213,10 +1213,23 @@ dmu_objset_sync_dnodes(multilist_sublist_t *list, dmu_tx_t *tx) + ASSERT3U(dn->dn_nlevels, <=, DN_MAX_LEVELS); + multilist_sublist_remove(list, dn); + ++ /* ++ * If we are not doing useraccounting (os_synced_dnodes == NULL) ++ * we are done with this dnode for this txg. Unset dn_dirty_txg ++ * if later txgs aren't dirtying it so that future holders do ++ * not get a stale value. Otherwise, we will do this in ++ * userquota_updates_task() when processing has completely ++ * finished for this txg. ++ */ + multilist_t *newlist = dn->dn_objset->os_synced_dnodes; + if (newlist != NULL) { + (void) dnode_add_ref(dn, newlist); + multilist_insert(newlist, dn); ++ } else { ++ mutex_enter(&dn->dn_mtx); ++ if (dn->dn_dirty_txg == tx->tx_txg) ++ dn->dn_dirty_txg = 0; ++ mutex_exit(&dn->dn_mtx); + } + + dnode_sync(dn, tx); +@@ -1621,6 +1634,8 @@ userquota_updates_task(void *arg) + dn->dn_id_flags |= DN_ID_CHKED_BONUS; + } + dn->dn_id_flags &= ~(DN_ID_NEW_EXIST); ++ if (dn->dn_dirty_txg == spa_syncing_txg(os->os_spa)) ++ dn->dn_dirty_txg = 0; + mutex_exit(&dn->dn_mtx); + + multilist_sublist_remove(list, dn); +diff --git a/module/zfs/dnode.c b/module/zfs/dnode.c +index d465b545..4a169c49 100644 +--- a/module/zfs/dnode.c ++++ b/module/zfs/dnode.c +@@ -137,7 +137,7 @@ dnode_cons(void *arg, void *unused, int kmflag) + bzero(&dn->dn_next_blksz[0], sizeof (dn->dn_next_blksz)); + + for (i = 0; i < TXG_SIZE; i++) { +- list_link_init(&dn->dn_dirty_link[i]); ++ multilist_link_init(&dn->dn_dirty_link[i]); + dn->dn_free_ranges[i] = NULL; + list_create(&dn->dn_dirty_records[i], + sizeof (dbuf_dirty_record_t), +@@ -147,6 +147,7 @@ dnode_cons(void *arg, void *unused, int kmflag) + dn->dn_allocated_txg = 0; + dn->dn_free_txg = 0; + dn->dn_assigned_txg = 0; ++ dn->dn_dirty_txg = 0; + dn->dn_dirtyctx = 0; + dn->dn_dirtyctx_firstset = NULL; + dn->dn_bonus = NULL; +@@ -184,7 +185,7 @@ dnode_dest(void *arg, void *unused) + ASSERT(!list_link_active(&dn->dn_link)); + + for (i = 0; i < TXG_SIZE; i++) { +- ASSERT(!list_link_active(&dn->dn_dirty_link[i])); ++ ASSERT(!multilist_link_active(&dn->dn_dirty_link[i])); + ASSERT3P(dn->dn_free_ranges[i], ==, NULL); + list_destroy(&dn->dn_dirty_records[i]); + ASSERT0(dn->dn_next_nblkptr[i]); +@@ -199,6 +200,7 @@ dnode_dest(void *arg, void *unused) + ASSERT0(dn->dn_allocated_txg); + ASSERT0(dn->dn_free_txg); + ASSERT0(dn->dn_assigned_txg); ++ ASSERT0(dn->dn_dirty_txg); + ASSERT0(dn->dn_dirtyctx); + ASSERT3P(dn->dn_dirtyctx_firstset, ==, NULL); + ASSERT3P(dn->dn_bonus, ==, NULL); +@@ -523,6 +525,7 @@ dnode_destroy(dnode_t *dn) + dn->dn_allocated_txg = 0; + dn->dn_free_txg = 0; + dn->dn_assigned_txg = 0; ++ dn->dn_dirty_txg = 0; + + dn->dn_dirtyctx = 0; + if (dn->dn_dirtyctx_firstset != NULL) { +@@ -592,6 +595,7 @@ dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs, + ASSERT0(dn->dn_maxblkid); + ASSERT0(dn->dn_allocated_txg); + ASSERT0(dn->dn_assigned_txg); ++ ASSERT0(dn->dn_dirty_txg); + ASSERT(refcount_is_zero(&dn->dn_tx_holds)); + ASSERT3U(refcount_count(&dn->dn_holds), <=, 1); + ASSERT(avl_is_empty(&dn->dn_dbufs)); +@@ -604,7 +608,7 @@ dnode_allocate(dnode_t *dn, dmu_object_type_t ot, int blocksize, int ibs, + ASSERT0(dn->dn_next_bonustype[i]); + ASSERT0(dn->dn_rm_spillblk[i]); + ASSERT0(dn->dn_next_blksz[i]); +- ASSERT(!list_link_active(&dn->dn_dirty_link[i])); ++ ASSERT(!multilist_link_active(&dn->dn_dirty_link[i])); + ASSERT3P(list_head(&dn->dn_dirty_records[i]), ==, NULL); + ASSERT3P(dn->dn_free_ranges[i], ==, NULL); + } +@@ -779,6 +783,7 @@ dnode_move_impl(dnode_t *odn, dnode_t *ndn) + ndn->dn_allocated_txg = odn->dn_allocated_txg; + ndn->dn_free_txg = odn->dn_free_txg; + ndn->dn_assigned_txg = odn->dn_assigned_txg; ++ ndn->dn_dirty_txg = odn->dn_dirty_txg; + ndn->dn_dirtyctx = odn->dn_dirtyctx; + ndn->dn_dirtyctx_firstset = odn->dn_dirtyctx_firstset; + ASSERT(refcount_count(&odn->dn_tx_holds) == 0); +@@ -845,6 +850,7 @@ dnode_move_impl(dnode_t *odn, dnode_t *ndn) + odn->dn_allocated_txg = 0; + odn->dn_free_txg = 0; + odn->dn_assigned_txg = 0; ++ odn->dn_dirty_txg = 0; + odn->dn_dirtyctx = 0; + odn->dn_dirtyctx_firstset = NULL; + odn->dn_have_spill = B_FALSE; +@@ -1069,6 +1075,10 @@ dnode_check_slots_free(dnode_children_t *children, int idx, int slots) + { + ASSERT3S(idx + slots, <=, DNODES_PER_BLOCK); + ++ /* ++ * If all dnode slots are either already free or ++ * evictable return B_TRUE. ++ */ + for (int i = idx; i < idx + slots; i++) { + dnode_handle_t *dnh = &children->dnc_children[i]; + dnode_t *dn = dnh->dnh_dnode; +@@ -1077,18 +1087,17 @@ dnode_check_slots_free(dnode_children_t *children, int idx, int slots) + continue; + } else if (DN_SLOT_IS_PTR(dn)) { + mutex_enter(&dn->dn_mtx); +- dmu_object_type_t type = dn->dn_type; ++ boolean_t can_free = (dn->dn_type == DMU_OT_NONE && ++ !DNODE_IS_DIRTY(dn)); + mutex_exit(&dn->dn_mtx); + +- if (type != DMU_OT_NONE) ++ if (!can_free) + return (B_FALSE); +- +- continue; ++ else ++ continue; + } else { + return (B_FALSE); + } +- +- return (B_FALSE); + } + + return (B_TRUE); +@@ -1594,7 +1603,7 @@ dnode_setdirty(dnode_t *dn, dmu_tx_t *tx) + /* + * If we are already marked dirty, we're done. + */ +- if (list_link_active(&dn->dn_dirty_link[txg & TXG_MASK])) { ++ if (multilist_link_active(&dn->dn_dirty_link[txg & TXG_MASK])) { + multilist_sublist_unlock(mls); + return; + } diff --git a/zfs-patches/0006-Reduce-taskq-and-context-switch-cost-of-zio-pipe.patch b/zfs-patches/0006-Reduce-taskq-and-context-switch-cost-of-zio-pipe.patch new file mode 100644 index 0000000..92dda45 --- /dev/null +++ b/zfs-patches/0006-Reduce-taskq-and-context-switch-cost-of-zio-pipe.patch @@ -0,0 +1,861 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Matthew Ahrens +Date: Thu, 2 Aug 2018 15:51:45 -0700 +Subject: [PATCH] Reduce taskq and context-switch cost of zio pipe + +When doing a read from disk, ZFS creates 3 ZIO's: a zio_null(), the +logical zio_read(), and then a physical zio. Currently, each of these +results in a separate taskq_dispatch(zio_execute). + +On high-read-iops workloads, this causes a significant performance +impact. By processing all 3 ZIO's in a single taskq entry, we reduce the +overhead on taskq locking and context switching. We accomplish this by +allowing zio_done() to return a "next zio to execute" to zio_execute(). + +This results in a ~12% performance increase for random reads, from +96,000 iops to 108,000 iops (with recordsize=8k, on SSD's). + +Reviewed by: Pavel Zakharov +Reviewed-by: Brian Behlendorf +Reviewed by: George Wilson +Signed-off-by: Matthew Ahrens +External-issue: DLPX-59292 +Requires-spl: spl-0.7-release +Closes #7736 + +(cherry-picked from behlendorf/issue-7736 496657ab3bcfeb638b1786e1759980ccfcacb08e) +Signed-off-by: Stoiko Ivanov +--- + include/sys/zio.h | 4 +- + module/zfs/zio.c | 250 +++++++++++++++++++++++++++++------------------------- + 2 files changed, 137 insertions(+), 117 deletions(-) + +diff --git a/include/sys/zio.h b/include/sys/zio.h +index 4b0eecc2..3618912c 100644 +--- a/include/sys/zio.h ++++ b/include/sys/zio.h +@@ -237,7 +237,7 @@ enum zio_child { + #define ZIO_CHILD_DDT_BIT ZIO_CHILD_BIT(ZIO_CHILD_DDT) + #define ZIO_CHILD_LOGICAL_BIT ZIO_CHILD_BIT(ZIO_CHILD_LOGICAL) + #define ZIO_CHILD_ALL_BITS \ +- (ZIO_CHILD_VDEV_BIT | ZIO_CHILD_GANG_BIT | \ ++ (ZIO_CHILD_VDEV_BIT | ZIO_CHILD_GANG_BIT | \ + ZIO_CHILD_DDT_BIT | ZIO_CHILD_LOGICAL_BIT) + + enum zio_wait_type { +@@ -375,7 +375,7 @@ typedef struct zio_transform { + struct zio_transform *zt_next; + } zio_transform_t; + +-typedef int zio_pipe_stage_t(zio_t *zio); ++typedef zio_t *zio_pipe_stage_t(zio_t *zio); + + /* + * The io_reexecute flags are distinct from io_flags because the child must +diff --git a/module/zfs/zio.c b/module/zfs/zio.c +index 9a465e1b..b08b4747 100644 +--- a/module/zfs/zio.c ++++ b/module/zfs/zio.c +@@ -75,9 +75,6 @@ uint64_t zio_buf_cache_frees[SPA_MAXBLOCKSIZE >> SPA_MINBLOCKSHIFT]; + + int zio_delay_max = ZIO_DELAY_MAX; + +-#define ZIO_PIPELINE_CONTINUE 0x100 +-#define ZIO_PIPELINE_STOP 0x101 +- + #define BP_SPANB(indblkshift, level) \ + (((uint64_t)1) << ((level) * ((indblkshift) - SPA_BLKPTRSHIFT))) + #define COMPARE_META_LEVEL 0x80000000ul +@@ -516,7 +513,8 @@ zio_wait_for_children(zio_t *zio, uint8_t childbits, enum zio_wait_type wait) + + __attribute__((always_inline)) + static inline void +-zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait) ++zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait, ++ zio_t **next_to_executep) + { + uint64_t *countp = &pio->io_children[zio->io_child_type][wait]; + int *errorp = &pio->io_child_error[zio->io_child_type]; +@@ -535,13 +533,33 @@ zio_notify_parent(zio_t *pio, zio_t *zio, enum zio_wait_type wait) + ZIO_TASKQ_INTERRUPT; + pio->io_stall = NULL; + mutex_exit(&pio->io_lock); ++ + /* +- * Dispatch the parent zio in its own taskq so that +- * the child can continue to make progress. This also +- * prevents overflowing the stack when we have deeply nested +- * parent-child relationships. ++ * If we can tell the caller to execute this parent next, do ++ * so. Otherwise dispatch the parent zio as its own task. ++ * ++ * Having the caller execute the parent when possible reduces ++ * locking on the zio taskq's, reduces context switch ++ * overhead, and has no recursion penalty. Note that one ++ * read from disk typically causes at least 3 zio's: a ++ * zio_null(), the logical zio_read(), and then a physical ++ * zio. When the physical ZIO completes, we are able to call ++ * zio_done() on all 3 of these zio's from one invocation of ++ * zio_execute() by returning the parent back to ++ * zio_execute(). Since the parent isn't executed until this ++ * thread returns back to zio_execute(), the caller should do ++ * so promptly. ++ * ++ * In other cases, dispatching the parent prevents ++ * overflowing the stack when we have deeply nested ++ * parent-child relationships, as we do with the "mega zio" ++ * of writes for spa_sync(), and the chain of ZIL blocks. + */ +- zio_taskq_dispatch(pio, type, B_FALSE); ++ if (next_to_executep != NULL && *next_to_executep == NULL) { ++ *next_to_executep = pio; ++ } else { ++ zio_taskq_dispatch(pio, type, B_FALSE); ++ } + } else { + mutex_exit(&pio->io_lock); + } +@@ -1187,7 +1205,7 @@ zio_shrink(zio_t *zio, uint64_t size) + * ========================================================================== + */ + +-static int ++static zio_t * + zio_read_bp_init(zio_t *zio) + { + blkptr_t *bp = zio->io_bp; +@@ -1221,15 +1239,15 @@ zio_read_bp_init(zio_t *zio) + if (BP_GET_DEDUP(bp) && zio->io_child_type == ZIO_CHILD_LOGICAL) + zio->io_pipeline = ZIO_DDT_READ_PIPELINE; + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + +-static int ++static zio_t * + zio_write_bp_init(zio_t *zio) + { + + if (!IO_IS_ALLOCATING(zio)) +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + + ASSERT(zio->io_child_type != ZIO_CHILD_DDT); + +@@ -1244,7 +1262,7 @@ zio_write_bp_init(zio_t *zio) + zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; + + if (BP_IS_EMBEDDED(bp)) +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + + /* + * If we've been overridden and nopwrite is set then +@@ -1255,13 +1273,13 @@ zio_write_bp_init(zio_t *zio) + ASSERT(!zp->zp_dedup); + ASSERT3U(BP_GET_CHECKSUM(bp), ==, zp->zp_checksum); + zio->io_flags |= ZIO_FLAG_NOPWRITE; +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + ASSERT(!zp->zp_nopwrite); + + if (BP_IS_HOLE(bp) || !zp->zp_dedup) +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + + ASSERT((zio_checksum_table[zp->zp_checksum].ci_flags & + ZCHECKSUM_FLAG_DEDUP) || zp->zp_dedup_verify); +@@ -1269,7 +1287,7 @@ zio_write_bp_init(zio_t *zio) + if (BP_GET_CHECKSUM(bp) == zp->zp_checksum) { + BP_SET_DEDUP(bp, 1); + zio->io_pipeline |= ZIO_STAGE_DDT_WRITE; +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + /* +@@ -1281,10 +1299,10 @@ zio_write_bp_init(zio_t *zio) + zio->io_pipeline = zio->io_orig_pipeline; + } + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + +-static int ++static zio_t * + zio_write_compress(zio_t *zio) + { + spa_t *spa = zio->io_spa; +@@ -1303,11 +1321,11 @@ zio_write_compress(zio_t *zio) + */ + if (zio_wait_for_children(zio, ZIO_CHILD_LOGICAL_BIT | + ZIO_CHILD_GANG_BIT, ZIO_WAIT_READY)) { +- return (ZIO_PIPELINE_STOP); ++ return (NULL); + } + + if (!IO_IS_ALLOCATING(zio)) +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + + if (zio->io_children_ready != NULL) { + /* +@@ -1366,7 +1384,7 @@ zio_write_compress(zio_t *zio) + zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; + ASSERT(spa_feature_is_active(spa, + SPA_FEATURE_EMBEDDED_DATA)); +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } else { + /* + * Round up compressed size up to the ashift +@@ -1459,10 +1477,10 @@ zio_write_compress(zio_t *zio) + zio->io_pipeline |= ZIO_STAGE_NOP_WRITE; + } + } +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + +-static int ++static zio_t * + zio_free_bp_init(zio_t *zio) + { + blkptr_t *bp = zio->io_bp; +@@ -1472,7 +1490,7 @@ zio_free_bp_init(zio_t *zio) + zio->io_pipeline = ZIO_DDT_FREE_PIPELINE; + } + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + /* +@@ -1541,12 +1559,12 @@ zio_taskq_member(zio_t *zio, zio_taskq_type_t q) + return (B_FALSE); + } + +-static int ++static zio_t * + zio_issue_async(zio_t *zio) + { + zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); + +- return (ZIO_PIPELINE_STOP); ++ return (NULL); + } + + void +@@ -1687,14 +1705,13 @@ __attribute__((always_inline)) + static inline void + __zio_execute(zio_t *zio) + { +- zio->io_executor = curthread; +- + ASSERT3U(zio->io_queued_timestamp, >, 0); + + while (zio->io_stage < ZIO_STAGE_DONE) { + enum zio_stage pipeline = zio->io_pipeline; + enum zio_stage stage = zio->io_stage; +- int rv; ++ ++ zio->io_executor = curthread; + + ASSERT(!MUTEX_HELD(&zio->io_lock)); + ASSERT(ISP2(stage)); +@@ -1736,12 +1753,16 @@ __zio_execute(zio_t *zio) + + zio->io_stage = stage; + zio->io_pipeline_trace |= zio->io_stage; +- rv = zio_pipeline[highbit64(stage) - 1](zio); + +- if (rv == ZIO_PIPELINE_STOP) +- return; ++ /* ++ * The zio pipeline stage returns the next zio to execute ++ * (typically the same as this one), or NULL if we should ++ * stop. ++ */ ++ zio = zio_pipeline[highbit64(stage) - 1](zio); + +- ASSERT(rv == ZIO_PIPELINE_CONTINUE); ++ if (zio == NULL) ++ return; + } + } + +@@ -2215,7 +2236,7 @@ zio_gang_tree_issue(zio_t *pio, zio_gang_node_t *gn, blkptr_t *bp, abd_t *data, + zio_nowait(zio); + } + +-static int ++static zio_t * + zio_gang_assemble(zio_t *zio) + { + blkptr_t *bp = zio->io_bp; +@@ -2227,16 +2248,16 @@ zio_gang_assemble(zio_t *zio) + + zio_gang_tree_assemble(zio, bp, &zio->io_gang_tree); + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + +-static int ++static zio_t * + zio_gang_issue(zio_t *zio) + { + blkptr_t *bp = zio->io_bp; + + if (zio_wait_for_children(zio, ZIO_CHILD_GANG_BIT, ZIO_WAIT_DONE)) { +- return (ZIO_PIPELINE_STOP); ++ return (NULL); + } + + ASSERT(BP_IS_GANG(bp) && zio->io_gang_leader == zio); +@@ -2250,7 +2271,7 @@ zio_gang_issue(zio_t *zio) + + zio->io_pipeline = ZIO_INTERLOCK_PIPELINE; + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + static void +@@ -2290,7 +2311,7 @@ zio_write_gang_done(zio_t *zio) + abd_put(zio->io_abd); + } + +-static int ++static zio_t * + zio_write_gang_block(zio_t *pio) + { + spa_t *spa = pio->io_spa; +@@ -2349,7 +2370,7 @@ zio_write_gang_block(zio_t *pio) + } + + pio->io_error = error; +- return (ZIO_PIPELINE_CONTINUE); ++ return (pio); + } + + if (pio == gio) { +@@ -2423,7 +2444,7 @@ zio_write_gang_block(zio_t *pio) + + zio_nowait(zio); + +- return (ZIO_PIPELINE_CONTINUE); ++ return (pio); + } + + /* +@@ -2444,7 +2465,7 @@ zio_write_gang_block(zio_t *pio) + * used for nopwrite, assuming that the salt and the checksums + * themselves remain secret. + */ +-static int ++static zio_t * + zio_nop_write(zio_t *zio) + { + blkptr_t *bp = zio->io_bp; +@@ -2471,7 +2492,7 @@ zio_nop_write(zio_t *zio) + BP_GET_COMPRESS(bp) != BP_GET_COMPRESS(bp_orig) || + BP_GET_DEDUP(bp) != BP_GET_DEDUP(bp_orig) || + zp->zp_copies != BP_GET_NDVAS(bp_orig)) +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + + /* + * If the checksums match then reset the pipeline so that we +@@ -2491,7 +2512,7 @@ zio_nop_write(zio_t *zio) + zio->io_flags |= ZIO_FLAG_NOPWRITE; + } + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + /* +@@ -2519,7 +2540,7 @@ zio_ddt_child_read_done(zio_t *zio) + mutex_exit(&pio->io_lock); + } + +-static int ++static zio_t * + zio_ddt_read_start(zio_t *zio) + { + blkptr_t *bp = zio->io_bp; +@@ -2540,7 +2561,7 @@ zio_ddt_read_start(zio_t *zio) + zio->io_vsd = dde; + + if (ddp_self == NULL) +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + + for (p = 0; p < DDT_PHYS_TYPES; p++, ddp++) { + if (ddp->ddp_phys_birth == 0 || ddp == ddp_self) +@@ -2553,23 +2574,23 @@ zio_ddt_read_start(zio_t *zio) + zio->io_priority, ZIO_DDT_CHILD_FLAGS(zio) | + ZIO_FLAG_DONT_PROPAGATE, &zio->io_bookmark)); + } +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + zio_nowait(zio_read(zio, zio->io_spa, bp, + zio->io_abd, zio->io_size, NULL, NULL, zio->io_priority, + ZIO_DDT_CHILD_FLAGS(zio), &zio->io_bookmark)); + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + +-static int ++static zio_t * + zio_ddt_read_done(zio_t *zio) + { + blkptr_t *bp = zio->io_bp; + + if (zio_wait_for_children(zio, ZIO_CHILD_DDT_BIT, ZIO_WAIT_DONE)) { +- return (ZIO_PIPELINE_STOP); ++ return (NULL); + } + + ASSERT(BP_GET_DEDUP(bp)); +@@ -2581,12 +2602,12 @@ zio_ddt_read_done(zio_t *zio) + ddt_entry_t *dde = zio->io_vsd; + if (ddt == NULL) { + ASSERT(spa_load_state(zio->io_spa) != SPA_LOAD_NONE); +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + if (dde == NULL) { + zio->io_stage = ZIO_STAGE_DDT_READ_START >> 1; + zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_FALSE); +- return (ZIO_PIPELINE_STOP); ++ return (NULL); + } + if (dde->dde_repair_abd != NULL) { + abd_copy(zio->io_abd, dde->dde_repair_abd, +@@ -2599,7 +2620,7 @@ zio_ddt_read_done(zio_t *zio) + + ASSERT(zio->io_vsd == NULL); + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + static boolean_t +@@ -2780,7 +2801,7 @@ zio_ddt_ditto_write_done(zio_t *zio) + ddt_exit(ddt); + } + +-static int ++static zio_t * + zio_ddt_write(zio_t *zio) + { + spa_t *spa = zio->io_spa; +@@ -2822,7 +2843,7 @@ zio_ddt_write(zio_t *zio) + } + zio->io_pipeline = ZIO_WRITE_PIPELINE; + ddt_exit(ddt); +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + ditto_copies = ddt_ditto_copies_needed(ddt, dde, ddp); +@@ -2848,7 +2869,7 @@ zio_ddt_write(zio_t *zio) + zio->io_bp_override = NULL; + BP_ZERO(bp); + ddt_exit(ddt); +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + dio = zio_write(zio, spa, txg, bp, zio->io_orig_abd, +@@ -2890,12 +2911,12 @@ zio_ddt_write(zio_t *zio) + if (dio) + zio_nowait(dio); + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + ddt_entry_t *freedde; /* for debugging */ + +-static int ++static zio_t * + zio_ddt_free(zio_t *zio) + { + spa_t *spa = zio->io_spa; +@@ -2916,7 +2937,7 @@ zio_ddt_free(zio_t *zio) + } + ddt_exit(ddt); + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + /* +@@ -2953,7 +2974,7 @@ zio_io_to_allocate(spa_t *spa) + return (zio); + } + +-static int ++static zio_t * + zio_dva_throttle(zio_t *zio) + { + spa_t *spa = zio->io_spa; +@@ -2963,7 +2984,7 @@ zio_dva_throttle(zio_t *zio) + !spa_normal_class(zio->io_spa)->mc_alloc_throttle_enabled || + zio->io_child_type == ZIO_CHILD_GANG || + zio->io_flags & ZIO_FLAG_NODATA) { +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + ASSERT(zio->io_child_type > ZIO_CHILD_GANG); +@@ -2979,22 +3000,7 @@ zio_dva_throttle(zio_t *zio) + nio = zio_io_to_allocate(zio->io_spa); + mutex_exit(&spa->spa_alloc_lock); + +- if (nio == zio) +- return (ZIO_PIPELINE_CONTINUE); +- +- if (nio != NULL) { +- ASSERT(nio->io_stage == ZIO_STAGE_DVA_THROTTLE); +- /* +- * We are passing control to a new zio so make sure that +- * it is processed by a different thread. We do this to +- * avoid stack overflows that can occur when parents are +- * throttled and children are making progress. We allow +- * it to go to the head of the taskq since it's already +- * been waiting. +- */ +- zio_taskq_dispatch(nio, ZIO_TASKQ_ISSUE, B_TRUE); +- } +- return (ZIO_PIPELINE_STOP); ++ return (nio); + } + + void +@@ -3013,7 +3019,7 @@ zio_allocate_dispatch(spa_t *spa) + zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, B_TRUE); + } + +-static int ++static zio_t * + zio_dva_allocate(zio_t *zio) + { + spa_t *spa = zio->io_spa; +@@ -3054,18 +3060,18 @@ zio_dva_allocate(zio_t *zio) + zio->io_error = error; + } + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + +-static int ++static zio_t * + zio_dva_free(zio_t *zio) + { + metaslab_free(zio->io_spa, zio->io_bp, zio->io_txg, B_FALSE); + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + +-static int ++static zio_t * + zio_dva_claim(zio_t *zio) + { + int error; +@@ -3074,7 +3080,7 @@ zio_dva_claim(zio_t *zio) + if (error) + zio->io_error = error; + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + /* +@@ -3172,7 +3178,7 @@ zio_free_zil(spa_t *spa, uint64_t txg, blkptr_t *bp) + * force the underlying vdev layers to call either zio_execute() or + * zio_interrupt() to ensure that the pipeline continues with the correct I/O. + */ +-static int ++static zio_t * + zio_vdev_io_start(zio_t *zio) + { + vdev_t *vd = zio->io_vd; +@@ -3192,7 +3198,7 @@ zio_vdev_io_start(zio_t *zio) + * The mirror_ops handle multiple DVAs in a single BP. + */ + vdev_mirror_ops.vdev_op_io_start(zio); +- return (ZIO_PIPELINE_STOP); ++ return (NULL); + } + + ASSERT3P(zio->io_logical, !=, zio); +@@ -3269,31 +3275,31 @@ zio_vdev_io_start(zio_t *zio) + !vdev_dtl_contains(vd, DTL_PARTIAL, zio->io_txg, 1)) { + ASSERT(zio->io_type == ZIO_TYPE_WRITE); + zio_vdev_io_bypass(zio); +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + if (vd->vdev_ops->vdev_op_leaf && + (zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE)) { + + if (zio->io_type == ZIO_TYPE_READ && vdev_cache_read(zio)) +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + + if ((zio = vdev_queue_io(zio)) == NULL) +- return (ZIO_PIPELINE_STOP); ++ return (NULL); + + if (!vdev_accessible(vd, zio)) { + zio->io_error = SET_ERROR(ENXIO); + zio_interrupt(zio); +- return (ZIO_PIPELINE_STOP); ++ return (NULL); + } + zio->io_delay = gethrtime(); + } + + vd->vdev_ops->vdev_op_io_start(zio); +- return (ZIO_PIPELINE_STOP); ++ return (NULL); + } + +-static int ++static zio_t * + zio_vdev_io_done(zio_t *zio) + { + vdev_t *vd = zio->io_vd; +@@ -3301,7 +3307,7 @@ zio_vdev_io_done(zio_t *zio) + boolean_t unexpected_error = B_FALSE; + + if (zio_wait_for_children(zio, ZIO_CHILD_VDEV_BIT, ZIO_WAIT_DONE)) { +- return (ZIO_PIPELINE_STOP); ++ return (NULL); + } + + ASSERT(zio->io_type == ZIO_TYPE_READ || zio->io_type == ZIO_TYPE_WRITE); +@@ -3337,7 +3343,7 @@ zio_vdev_io_done(zio_t *zio) + if (unexpected_error) + VERIFY(vdev_probe(vd, zio) == NULL); + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + /* +@@ -3366,13 +3372,13 @@ zio_vsd_default_cksum_report(zio_t *zio, zio_cksum_report_t *zcr, void *ignored) + zcr->zcr_free = zio_abd_free; + } + +-static int ++static zio_t * + zio_vdev_io_assess(zio_t *zio) + { + vdev_t *vd = zio->io_vd; + + if (zio_wait_for_children(zio, ZIO_CHILD_VDEV_BIT, ZIO_WAIT_DONE)) { +- return (ZIO_PIPELINE_STOP); ++ return (NULL); + } + + if (vd == NULL && !(zio->io_flags & ZIO_FLAG_CONFIG_WRITER)) +@@ -3402,7 +3408,7 @@ zio_vdev_io_assess(zio_t *zio) + zio->io_stage = ZIO_STAGE_VDEV_IO_START >> 1; + zio_taskq_dispatch(zio, ZIO_TASKQ_ISSUE, + zio_requeue_io_start_cut_in_line); +- return (ZIO_PIPELINE_STOP); ++ return (NULL); + } + + /* +@@ -3442,7 +3448,7 @@ zio_vdev_io_assess(zio_t *zio) + zio->io_physdone(zio->io_logical); + } + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + void +@@ -3477,7 +3483,7 @@ zio_vdev_io_bypass(zio_t *zio) + * Generate and verify checksums + * ========================================================================== + */ +-static int ++static zio_t * + zio_checksum_generate(zio_t *zio) + { + blkptr_t *bp = zio->io_bp; +@@ -3491,7 +3497,7 @@ zio_checksum_generate(zio_t *zio) + checksum = zio->io_prop.zp_checksum; + + if (checksum == ZIO_CHECKSUM_OFF) +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + + ASSERT(checksum == ZIO_CHECKSUM_LABEL); + } else { +@@ -3505,10 +3511,10 @@ zio_checksum_generate(zio_t *zio) + + zio_checksum_compute(zio, checksum, zio->io_abd, zio->io_size); + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + +-static int ++static zio_t * + zio_checksum_verify(zio_t *zio) + { + zio_bad_cksum_t info; +@@ -3523,7 +3529,7 @@ zio_checksum_verify(zio_t *zio) + * We're either verifying a label checksum, or nothing at all. + */ + if (zio->io_prop.zp_checksum == ZIO_CHECKSUM_OFF) +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + + ASSERT(zio->io_prop.zp_checksum == ZIO_CHECKSUM_LABEL); + } +@@ -3538,7 +3544,7 @@ zio_checksum_verify(zio_t *zio) + } + } + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + /* +@@ -3581,7 +3587,7 @@ zio_worst_error(int e1, int e2) + * I/O completion + * ========================================================================== + */ +-static int ++static zio_t * + zio_ready(zio_t *zio) + { + blkptr_t *bp = zio->io_bp; +@@ -3590,7 +3596,7 @@ zio_ready(zio_t *zio) + + if (zio_wait_for_children(zio, ZIO_CHILD_GANG_BIT | ZIO_CHILD_DDT_BIT, + ZIO_WAIT_READY)) { +- return (ZIO_PIPELINE_STOP); ++ return (NULL); + } + + if (zio->io_ready) { +@@ -3636,7 +3642,7 @@ zio_ready(zio_t *zio) + */ + for (; pio != NULL; pio = pio_next) { + pio_next = zio_walk_parents(zio, &zl); +- zio_notify_parent(pio, zio, ZIO_WAIT_READY); ++ zio_notify_parent(pio, zio, ZIO_WAIT_READY, NULL); + } + + if (zio->io_flags & ZIO_FLAG_NODATA) { +@@ -3652,7 +3658,7 @@ zio_ready(zio_t *zio) + zio->io_spa->spa_syncing_txg == zio->io_txg) + zio_handle_ignored_writes(zio); + +- return (ZIO_PIPELINE_CONTINUE); ++ return (zio); + } + + /* +@@ -3716,7 +3722,7 @@ zio_dva_throttle_done(zio_t *zio) + zio_allocate_dispatch(zio->io_spa); + } + +-static int ++static zio_t * + zio_done(zio_t *zio) + { + /* +@@ -3733,7 +3739,7 @@ zio_done(zio_t *zio) + * wait for them and then repeat this pipeline stage. + */ + if (zio_wait_for_children(zio, ZIO_CHILD_ALL_BITS, ZIO_WAIT_DONE)) { +- return (ZIO_PIPELINE_STOP); ++ return (NULL); + } + + /* +@@ -3957,7 +3963,12 @@ zio_done(zio_t *zio) + if ((pio->io_flags & ZIO_FLAG_GODFATHER) && + (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND)) { + zio_remove_child(pio, zio, remove_zl); +- zio_notify_parent(pio, zio, ZIO_WAIT_DONE); ++ /* ++ * This is a rare code path, so we don't ++ * bother with "next_to_execute". ++ */ ++ zio_notify_parent(pio, zio, ZIO_WAIT_DONE, ++ NULL); + } + } + +@@ -3969,7 +3980,11 @@ zio_done(zio_t *zio) + */ + ASSERT(!(zio->io_flags & ZIO_FLAG_GODFATHER)); + zio->io_flags |= ZIO_FLAG_DONT_PROPAGATE; +- zio_notify_parent(pio, zio, ZIO_WAIT_DONE); ++ /* ++ * This is a rare code path, so we don't bother with ++ * "next_to_execute". ++ */ ++ zio_notify_parent(pio, zio, ZIO_WAIT_DONE, NULL); + } else if (zio->io_reexecute & ZIO_REEXECUTE_SUSPEND) { + /* + * We'd fail again if we reexecuted now, so suspend +@@ -3987,7 +4002,7 @@ zio_done(zio_t *zio) + (task_func_t *)zio_reexecute, zio, 0, + &zio->io_tqent); + } +- return (ZIO_PIPELINE_STOP); ++ return (NULL); + } + + ASSERT(zio->io_child_count == 0); +@@ -4023,12 +4038,17 @@ zio_done(zio_t *zio) + zio->io_state[ZIO_WAIT_DONE] = 1; + mutex_exit(&zio->io_lock); + ++ /* ++ * We are done executing this zio. We may want to execute a parent ++ * next. See the comment in zio_notify_parent(). ++ */ ++ zio_t *next_to_execute = NULL; + zl = NULL; + for (pio = zio_walk_parents(zio, &zl); pio != NULL; pio = pio_next) { + zio_link_t *remove_zl = zl; + pio_next = zio_walk_parents(zio, &zl); + zio_remove_child(pio, zio, remove_zl); +- zio_notify_parent(pio, zio, ZIO_WAIT_DONE); ++ zio_notify_parent(pio, zio, ZIO_WAIT_DONE, &next_to_execute); + } + + if (zio->io_waiter != NULL) { +@@ -4040,7 +4060,7 @@ zio_done(zio_t *zio) + zio_destroy(zio); + } + +- return (ZIO_PIPELINE_STOP); ++ return (next_to_execute); + } + + /* diff --git a/zfs-patches/series b/zfs-patches/series index 5130acc..3b4626f 100644 --- a/zfs-patches/series +++ b/zfs-patches/series @@ -2,3 +2,5 @@ 0002-import-with-d-dev-disk-by-id-in-scan-service.patch 0003-always-load-ZFS-module-on-boot.patch 0004-Fix-deadlock-between-zfs-umount-snapentry_expire.patch +0005-Fix-race-in-dnode_check_slots_free.patch +0006-Reduce-taskq-and-context-switch-cost-of-zio-pipe.patch