From b8d06fca089fae4680c3a552fc55c512bfb02202 Mon Sep 17 00:00:00 2001 From: Richard Yao Date: Mon, 7 May 2012 13:49:51 -0400 Subject: [PATCH] Switch KM_SLEEP to KM_PUSHPAGE Differences between how paging is done on Solaris and Linux can cause deadlocks if KM_SLEEP is used in any the following contexts. * The txg_sync thread * The zvol write/discard threads * The zpl_putpage() VFS callback This is because KM_SLEEP will allow for direct reclaim which may result in the VM calling back in to the filesystem or block layer to write out pages. If a lock is held over this operation the potential exists to deadlock the system. To ensure forward progress all memory allocations in these contexts must us KM_PUSHPAGE which disables performing any I/O to accomplish the memory allocation. Previously, this behavior was acheived by setting PF_MEMALLOC on the thread. However, that resulted in unexpected side effects such as the exhaustion of pages in ZONE_DMA. This approach touchs more of the zfs code, but it is more consistent with the right way to handle these cases under Linux. This is patch lays the ground work for being able to safely revert the following commits which used PF_MEMALLOC: 21ade34 Disable direct reclaim for z_wr_* threads cfc9a5c Fix zpl_writepage() deadlock eec8164 Fix ASSERTION(!dsl_pool_sync_context(tx->tx_pool)) Signed-off-by: Richard Yao Signed-off-by: Brian Behlendorf Issue #726 --- include/sys/dbuf.h | 14 +++---- include/sys/dsl_dataset.h | 2 +- include/sys/dsl_dir.h | 2 +- include/sys/spa.h | 12 +++--- module/zcommon/zprop_common.c | 2 +- module/zfs/arc.c | 2 +- module/zfs/bplist.c | 2 +- module/zfs/dbuf.c | 6 +-- module/zfs/ddt.c | 2 +- module/zfs/dmu.c | 12 +++--- module/zfs/dmu_objset.c | 2 +- module/zfs/dmu_traverse.c | 6 +-- module/zfs/dmu_tx.c | 6 +-- module/zfs/dmu_zfetch.c | 2 +- module/zfs/dnode.c | 6 +-- module/zfs/dsl_dataset.c | 2 +- module/zfs/dsl_deadlist.c | 6 ++- module/zfs/dsl_dir.c | 10 ++--- module/zfs/dsl_prop.c | 6 +-- module/zfs/lzjb.c | 2 +- module/zfs/metaslab.c | 10 ++--- module/zfs/refcount.c | 4 +- module/zfs/sa.c | 26 ++++++------ module/zfs/spa.c | 74 +++++++++++++++++------------------ module/zfs/spa_config.c | 24 ++++++------ module/zfs/spa_history.c | 10 ++--- module/zfs/spa_misc.c | 10 ++--- module/zfs/space_map.c | 6 +-- module/zfs/txg.c | 2 +- module/zfs/unique.c | 2 +- module/zfs/vdev.c | 10 ++--- module/zfs/vdev_cache.c | 4 +- module/zfs/vdev_disk.c | 6 +-- module/zfs/vdev_file.c | 2 +- module/zfs/vdev_label.c | 16 ++++---- module/zfs/vdev_mirror.c | 4 +- module/zfs/vdev_raidz.c | 6 +-- module/zfs/zap.c | 8 ++-- module/zfs/zap_micro.c | 6 +-- module/zfs/zfs_acl.c | 6 +-- module/zfs/zfs_fm.c | 6 +-- module/zfs/zfs_rlock.c | 2 +- module/zfs/zfs_znode.c | 8 ++-- module/zfs/zil.c | 14 +++---- module/zfs/zvol.c | 2 +- 45 files changed, 187 insertions(+), 185 deletions(-) diff --git a/include/sys/dbuf.h b/include/sys/dbuf.h index f3e14aad3..ef93eb54c 100644 --- a/include/sys/dbuf.h +++ b/include/sys/dbuf.h @@ -345,13 +345,13 @@ boolean_t dbuf_is_metadata(dmu_buf_impl_t *db); } \ _NOTE(CONSTCOND) } while (0) -#define dprintf_dbuf_bp(db, bp, fmt, ...) do { \ - if (zfs_flags & ZFS_DEBUG_DPRINTF) { \ - char *__blkbuf = kmem_alloc(BP_SPRINTF_LEN, KM_SLEEP); \ - sprintf_blkptr(__blkbuf, bp); \ - dprintf_dbuf(db, fmt " %s\n", __VA_ARGS__, __blkbuf); \ - kmem_free(__blkbuf, BP_SPRINTF_LEN); \ - } \ +#define dprintf_dbuf_bp(db, bp, fmt, ...) do { \ + if (zfs_flags & ZFS_DEBUG_DPRINTF) { \ + char *__blkbuf = kmem_alloc(BP_SPRINTF_LEN, KM_PUSHPAGE); \ + sprintf_blkptr(__blkbuf, bp); \ + dprintf_dbuf(db, fmt " %s\n", __VA_ARGS__, __blkbuf); \ + kmem_free(__blkbuf, BP_SPRINTF_LEN); \ + } \ _NOTE(CONSTCOND) } while (0) #define DBUF_VERIFY(db) dbuf_verify(db) diff --git a/include/sys/dsl_dataset.h b/include/sys/dsl_dataset.h index c4530a8f0..948abb020 100644 --- a/include/sys/dsl_dataset.h +++ b/include/sys/dsl_dataset.h @@ -271,7 +271,7 @@ int dsl_destroy_inconsistent(const char *dsname, void *arg); #ifdef ZFS_DEBUG #define dprintf_ds(ds, fmt, ...) do { \ if (zfs_flags & ZFS_DEBUG_DPRINTF) { \ - char *__ds_name = kmem_alloc(MAXNAMELEN, KM_SLEEP); \ + char *__ds_name = kmem_alloc(MAXNAMELEN, KM_PUSHPAGE); \ dsl_dataset_name(ds, __ds_name); \ dprintf("ds=%s " fmt, __ds_name, __VA_ARGS__); \ kmem_free(__ds_name, MAXNAMELEN); \ diff --git a/include/sys/dsl_dir.h b/include/sys/dsl_dir.h index 2191635dd..65ad202bb 100644 --- a/include/sys/dsl_dir.h +++ b/include/sys/dsl_dir.h @@ -150,7 +150,7 @@ timestruc_t dsl_dir_snap_cmtime(dsl_dir_t *dd); #define dprintf_dd(dd, fmt, ...) do { \ if (zfs_flags & ZFS_DEBUG_DPRINTF) { \ char *__ds_name = kmem_alloc(MAXNAMELEN + strlen(MOS_DIR_NAME) + 1, \ - KM_SLEEP); \ + KM_PUSHPAGE); \ dsl_dir_name(dd, __ds_name); \ dprintf("dd=%s " fmt, __ds_name, __VA_ARGS__); \ kmem_free(__ds_name, MAXNAMELEN + strlen(MOS_DIR_NAME) + 1); \ diff --git a/include/sys/spa.h b/include/sys/spa.h index 9de5736ed..28bb4e1de 100644 --- a/include/sys/spa.h +++ b/include/sys/spa.h @@ -690,12 +690,12 @@ extern void spa_configfile_set(spa_t *, nvlist_t *, boolean_t); extern void spa_event_notify(spa_t *spa, vdev_t *vdev, const char *name); #ifdef ZFS_DEBUG -#define dprintf_bp(bp, fmt, ...) do { \ - if (zfs_flags & ZFS_DEBUG_DPRINTF) { \ - char *__blkbuf = kmem_alloc(BP_SPRINTF_LEN, KM_SLEEP); \ - sprintf_blkptr(__blkbuf, (bp)); \ - dprintf(fmt " %s\n", __VA_ARGS__, __blkbuf); \ - kmem_free(__blkbuf, BP_SPRINTF_LEN); \ +#define dprintf_bp(bp, fmt, ...) do { \ + if (zfs_flags & ZFS_DEBUG_DPRINTF) { \ + char *__blkbuf = kmem_alloc(BP_SPRINTF_LEN, KM_PUSHPAGE); \ + sprintf_blkptr(__blkbuf, (bp)); \ + dprintf(fmt " %s\n", __VA_ARGS__, __blkbuf); \ + kmem_free(__blkbuf, BP_SPRINTF_LEN); \ } \ _NOTE(CONSTCOND) } while (0) #else diff --git a/module/zcommon/zprop_common.c b/module/zcommon/zprop_common.c index ab5b4662b..0a0af2334 100644 --- a/module/zcommon/zprop_common.c +++ b/module/zcommon/zprop_common.c @@ -171,7 +171,7 @@ zprop_iter_common(zprop_func func, void *cb, boolean_t show_all, size = num_props * sizeof (zprop_desc_t *); #if defined(_KERNEL) - order = kmem_alloc(size, KM_SLEEP); + order = kmem_alloc(size, KM_PUSHPAGE); #else if ((order = malloc(size)) == NULL) return (ZPROP_CONT); diff --git a/module/zfs/arc.c b/module/zfs/arc.c index aad37a39b..e75a3f59a 100644 --- a/module/zfs/arc.c +++ b/module/zfs/arc.c @@ -3547,7 +3547,7 @@ arc_write(zio_t *pio, spa_t *spa, uint64_t txg, ASSERT(hdr->b_acb == NULL); if (l2arc) hdr->b_flags |= ARC_L2CACHE; - callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_SLEEP); + callback = kmem_zalloc(sizeof (arc_write_callback_t), KM_PUSHPAGE); callback->awcb_ready = ready; callback->awcb_done = done; callback->awcb_private = private; diff --git a/module/zfs/bplist.c b/module/zfs/bplist.c index 5d1cf7e76..d196351dc 100644 --- a/module/zfs/bplist.c +++ b/module/zfs/bplist.c @@ -44,7 +44,7 @@ bplist_destroy(bplist_t *bpl) void bplist_append(bplist_t *bpl, const blkptr_t *bp) { - bplist_entry_t *bpe = kmem_alloc(sizeof (*bpe), KM_SLEEP); + bplist_entry_t *bpe = kmem_alloc(sizeof (*bpe), KM_PUSHPAGE); mutex_enter(&bpl->bpl_lock); bpe->bpe_blk = *bp; diff --git a/module/zfs/dbuf.c b/module/zfs/dbuf.c index 42d82bbfb..1f6fa9340 100644 --- a/module/zfs/dbuf.c +++ b/module/zfs/dbuf.c @@ -298,7 +298,7 @@ retry: #if defined(_KERNEL) && defined(HAVE_SPL) /* Large allocations which do not require contiguous pages * should be using vmem_alloc() in the linux kernel */ - h->hash_table = vmem_zalloc(hsize * sizeof (void *), KM_SLEEP); + h->hash_table = vmem_zalloc(hsize * sizeof (void *), KM_PUSHPAGE); #else h->hash_table = kmem_zalloc(hsize * sizeof (void *), KM_NOSLEEP); #endif @@ -1719,7 +1719,7 @@ dbuf_create(dnode_t *dn, uint8_t level, uint64_t blkid, ASSERT(RW_LOCK_HELD(&dn->dn_struct_rwlock)); ASSERT(dn->dn_type != DMU_OT_NONE); - db = kmem_cache_alloc(dbuf_cache, KM_SLEEP); + db = kmem_cache_alloc(dbuf_cache, KM_PUSHPAGE); db->db_objset = os; db->db.db_object = dn->dn_object; @@ -2019,7 +2019,7 @@ dbuf_hold_impl(dnode_t *dn, uint8_t level, uint64_t blkid, int fail_sparse, int error; dh = kmem_zalloc(sizeof(struct dbuf_hold_impl_data) * - DBUF_HOLD_IMPL_MAX_DEPTH, KM_SLEEP); + DBUF_HOLD_IMPL_MAX_DEPTH, KM_PUSHPAGE); __dbuf_hold_impl_init(dh, dn, level, blkid, fail_sparse, tag, dbp, 0); error = __dbuf_hold_impl(dh); diff --git a/module/zfs/ddt.c b/module/zfs/ddt.c index 18c6cef5a..f93024d8f 100644 --- a/module/zfs/ddt.c +++ b/module/zfs/ddt.c @@ -504,7 +504,7 @@ ddt_get_dedup_stats(spa_t *spa, ddt_stat_t *dds_total) ddt_histogram_t *ddh_total; /* XXX: Move to a slab */ - ddh_total = kmem_zalloc(sizeof (ddt_histogram_t), KM_SLEEP); + ddh_total = kmem_zalloc(sizeof (ddt_histogram_t), KM_PUSHPAGE); ddt_get_dedup_histogram(spa, ddh_total); ddt_histogram_stat(dds_total, ddh_total); kmem_free(ddh_total, sizeof (ddt_histogram_t)); diff --git a/module/zfs/dmu.c b/module/zfs/dmu.c index cda4f8428..1d4d1257d 100644 --- a/module/zfs/dmu.c +++ b/module/zfs/dmu.c @@ -381,7 +381,7 @@ dmu_buf_hold_array_by_dnode(dnode_t *dn, uint64_t offset, uint64_t length, } nblks = 1; } - dbp = kmem_zalloc(sizeof (dmu_buf_t *) * nblks, KM_SLEEP | KM_NODEBUG); + dbp = kmem_zalloc(sizeof (dmu_buf_t *) * nblks, KM_PUSHPAGE | KM_NODEBUG); if (dn->dn_objset->os_dsl_dataset) dp = dn->dn_objset->os_dsl_dataset->ds_dir->dd_pool; @@ -863,11 +863,11 @@ dmu_xuio_init(xuio_t *xuio, int nblk) uio_t *uio = &xuio->xu_uio; uio->uio_iovcnt = nblk; - uio->uio_iov = kmem_zalloc(nblk * sizeof (iovec_t), KM_SLEEP); + uio->uio_iov = kmem_zalloc(nblk * sizeof (iovec_t), KM_PUSHPAGE); - priv = kmem_zalloc(sizeof (dmu_xuio_t), KM_SLEEP); + priv = kmem_zalloc(sizeof (dmu_xuio_t), KM_PUSHPAGE); priv->cnt = nblk; - priv->bufs = kmem_zalloc(nblk * sizeof (arc_buf_t *), KM_SLEEP); + priv->bufs = kmem_zalloc(nblk * sizeof (arc_buf_t *), KM_PUSHPAGE); priv->iovp = uio->uio_iov; XUIO_XUZC_PRIV(xuio) = priv; @@ -1431,7 +1431,7 @@ dmu_sync_late_arrival(zio_t *pio, objset_t *os, dmu_sync_cb_t *done, zgd_t *zgd, return (EIO); /* Make zl_get_data do txg_waited_synced() */ } - dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP); + dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_PUSHPAGE); dsa->dsa_dr = NULL; dsa->dsa_done = done; dsa->dsa_zgd = zgd; @@ -1555,7 +1555,7 @@ dmu_sync(zio_t *pio, uint64_t txg, dmu_sync_cb_t *done, zgd_t *zgd) dr->dt.dl.dr_override_state = DR_IN_DMU_SYNC; mutex_exit(&db->db_mtx); - dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_SLEEP); + dsa = kmem_alloc(sizeof (dmu_sync_arg_t), KM_PUSHPAGE); dsa->dsa_dr = dr; dsa->dsa_done = done; dsa->dsa_zgd = zgd; diff --git a/module/zfs/dmu_objset.c b/module/zfs/dmu_objset.c index 4e043243c..a34584ebf 100644 --- a/module/zfs/dmu_objset.c +++ b/module/zfs/dmu_objset.c @@ -262,7 +262,7 @@ dmu_objset_open_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *bp, ASSERT(ds == NULL || MUTEX_HELD(&ds->ds_opening_lock)); - os = kmem_zalloc(sizeof (objset_t), KM_SLEEP); + os = kmem_zalloc(sizeof (objset_t), KM_PUSHPAGE); os->os_dsl_dataset = ds; os->os_spa = spa; os->os_rootbp = bp; diff --git a/module/zfs/dmu_traverse.c b/module/zfs/dmu_traverse.c index 998cb4b6a..376f60f82 100644 --- a/module/zfs/dmu_traverse.c +++ b/module/zfs/dmu_traverse.c @@ -361,9 +361,9 @@ traverse_impl(spa_t *spa, dsl_dataset_t *ds, blkptr_t *rootbp, zbookmark_t *czb; int err; - td = kmem_alloc(sizeof(traverse_data_t), KM_SLEEP); - pd = kmem_zalloc(sizeof(prefetch_data_t), KM_SLEEP); - czb = kmem_alloc(sizeof(zbookmark_t), KM_SLEEP); + td = kmem_alloc(sizeof(traverse_data_t), KM_PUSHPAGE); + pd = kmem_zalloc(sizeof(prefetch_data_t), KM_PUSHPAGE); + czb = kmem_alloc(sizeof(zbookmark_t), KM_PUSHPAGE); td->td_spa = spa; td->td_objset = ds ? ds->ds_object : 0; diff --git a/module/zfs/dmu_tx.c b/module/zfs/dmu_tx.c index ead0f3e2a..81c6dfea2 100644 --- a/module/zfs/dmu_tx.c +++ b/module/zfs/dmu_tx.c @@ -63,7 +63,7 @@ static kstat_t *dmu_tx_ksp; dmu_tx_t * dmu_tx_create_dd(dsl_dir_t *dd) { - dmu_tx_t *tx = kmem_zalloc(sizeof (dmu_tx_t), KM_SLEEP); + dmu_tx_t *tx = kmem_zalloc(sizeof (dmu_tx_t), KM_PUSHPAGE); tx->tx_dir = dd; if (dd) tx->tx_pool = dd->dd_pool; @@ -141,7 +141,7 @@ dmu_tx_hold_object_impl(dmu_tx_t *tx, objset_t *os, uint64_t object, } } - txh = kmem_zalloc(sizeof (dmu_tx_hold_t), KM_SLEEP); + txh = kmem_zalloc(sizeof (dmu_tx_hold_t), KM_PUSHPAGE); txh->txh_tx = tx; txh->txh_dnode = dn; #ifdef DEBUG_DMU_TX @@ -1241,7 +1241,7 @@ dmu_tx_callback_register(dmu_tx_t *tx, dmu_tx_callback_func_t *func, void *data) { dmu_tx_callback_t *dcb; - dcb = kmem_alloc(sizeof (dmu_tx_callback_t), KM_SLEEP); + dcb = kmem_alloc(sizeof (dmu_tx_callback_t), KM_PUSHPAGE); dcb->dcb_func = func; dcb->dcb_data = data; diff --git a/module/zfs/dmu_zfetch.c b/module/zfs/dmu_zfetch.c index 897ea8adb..1763bae51 100644 --- a/module/zfs/dmu_zfetch.c +++ b/module/zfs/dmu_zfetch.c @@ -699,7 +699,7 @@ dmu_zfetch(zfetch_t *zf, uint64_t offset, uint64_t size, int prefetched) if (cur_streams >= max_streams) { return; } - newstream = kmem_zalloc(sizeof (zstream_t), KM_SLEEP); + newstream = kmem_zalloc(sizeof (zstream_t), KM_PUSHPAGE); } newstream->zst_offset = zst.zst_offset; diff --git a/module/zfs/dnode.c b/module/zfs/dnode.c index 5438f60d0..99ac62565 100644 --- a/module/zfs/dnode.c +++ b/module/zfs/dnode.c @@ -372,7 +372,7 @@ static dnode_t * dnode_create(objset_t *os, dnode_phys_t *dnp, dmu_buf_impl_t *db, uint64_t object, dnode_handle_t *dnh) { - dnode_t *dn = kmem_cache_alloc(dnode_cache, KM_SLEEP); + dnode_t *dn = kmem_cache_alloc(dnode_cache, KM_PUSHPAGE); ASSERT(!POINTER_IS_VALID(dn->dn_objset)); dn->dn_moved = 0; @@ -1491,7 +1491,7 @@ dnode_clear_range(dnode_t *dn, uint64_t blkid, uint64_t nblks, dmu_tx_t *tx) } else if (blkid > rp->fr_blkid && endblk < fr_endblk) { /* clear a chunk out of this range */ free_range_t *new_rp = - kmem_alloc(sizeof (free_range_t), KM_SLEEP); + kmem_alloc(sizeof (free_range_t), KM_PUSHPAGE); new_rp->fr_blkid = endblk; new_rp->fr_nblks = fr_endblk - endblk; @@ -1669,7 +1669,7 @@ done: avl_tree_t *tree = &dn->dn_ranges[tx->tx_txg&TXG_MASK]; /* Add new range to dn_ranges */ - rp = kmem_alloc(sizeof (free_range_t), KM_SLEEP); + rp = kmem_alloc(sizeof (free_range_t), KM_PUSHPAGE); rp->fr_blkid = blkid; rp->fr_nblks = nblks; found = avl_find(tree, rp, &where); diff --git a/module/zfs/dsl_dataset.c b/module/zfs/dsl_dataset.c index fce6d3c1a..e7cbc5138 100644 --- a/module/zfs/dsl_dataset.c +++ b/module/zfs/dsl_dataset.c @@ -390,7 +390,7 @@ dsl_dataset_get_ref(dsl_pool_t *dp, uint64_t dsobj, void *tag, if (ds == NULL) { dsl_dataset_t *winner = NULL; - ds = kmem_zalloc(sizeof (dsl_dataset_t), KM_SLEEP); + ds = kmem_zalloc(sizeof (dsl_dataset_t), KM_PUSHPAGE); ds->ds_dbuf = dbuf; ds->ds_object = dsobj; ds->ds_phys = dbuf->db_data; diff --git a/module/zfs/dsl_deadlist.c b/module/zfs/dsl_deadlist.c index dd6db2120..1e89a68d7 100644 --- a/module/zfs/dsl_deadlist.c +++ b/module/zfs/dsl_deadlist.c @@ -80,7 +80,9 @@ dsl_deadlist_load_tree(dsl_deadlist_t *dl) for (zap_cursor_init(&zc, dl->dl_os, dl->dl_object); zap_cursor_retrieve(&zc, &za) == 0; zap_cursor_advance(&zc)) { - dsl_deadlist_entry_t *dle = kmem_alloc(sizeof (*dle), KM_SLEEP); + dsl_deadlist_entry_t *dle; + + dle = kmem_alloc(sizeof (*dle), KM_PUSHPAGE); dle->dle_mintxg = strtonum(za.za_name, NULL); VERIFY3U(0, ==, bpobj_open(&dle->dle_bpobj, dl->dl_os, za.za_first_integer)); @@ -215,7 +217,7 @@ dsl_deadlist_add_key(dsl_deadlist_t *dl, uint64_t mintxg, dmu_tx_t *tx) dsl_deadlist_load_tree(dl); - dle = kmem_alloc(sizeof (*dle), KM_SLEEP); + dle = kmem_alloc(sizeof (*dle), KM_PUSHPAGE); dle->dle_mintxg = mintxg; obj = bpobj_alloc(dl->dl_os, SPA_MAXBLOCKSIZE, tx); VERIFY3U(0, ==, bpobj_open(&dle->dle_bpobj, dl->dl_os, obj)); diff --git a/module/zfs/dsl_dir.c b/module/zfs/dsl_dir.c index 224cc0a3c..d615832c5 100644 --- a/module/zfs/dsl_dir.c +++ b/module/zfs/dsl_dir.c @@ -96,7 +96,7 @@ dsl_dir_open_obj(dsl_pool_t *dp, uint64_t ddobj, if (dd == NULL) { dsl_dir_t *winner; - dd = kmem_zalloc(sizeof (dsl_dir_t), KM_SLEEP); + dd = kmem_zalloc(sizeof (dsl_dir_t), KM_PUSHPAGE); dd->dd_object = ddobj; dd->dd_dbuf = dbuf; dd->dd_pool = dp; @@ -791,7 +791,7 @@ dsl_dir_tempreserve_impl(dsl_dir_t *dd, uint64_t asize, boolean_t netfree, asize - ref_rsrv); mutex_exit(&dd->dd_lock); - tr = kmem_zalloc(sizeof (struct tempreserve), KM_SLEEP); + tr = kmem_zalloc(sizeof (struct tempreserve), KM_PUSHPAGE); tr->tr_ds = dd; tr->tr_size = asize; list_insert_tail(tr_list, tr); @@ -825,7 +825,7 @@ dsl_dir_tempreserve_space(dsl_dir_t *dd, uint64_t lsize, uint64_t asize, return (0); } - tr_list = kmem_alloc(sizeof (list_t), KM_SLEEP); + tr_list = kmem_alloc(sizeof (list_t), KM_PUSHPAGE); list_create(tr_list, sizeof (struct tempreserve), offsetof(struct tempreserve, tr_node)); ASSERT3S(asize, >, 0); @@ -835,7 +835,7 @@ dsl_dir_tempreserve_space(dsl_dir_t *dd, uint64_t lsize, uint64_t asize, if (err == 0) { struct tempreserve *tr; - tr = kmem_zalloc(sizeof (struct tempreserve), KM_SLEEP); + tr = kmem_zalloc(sizeof (struct tempreserve), KM_PUSHPAGE); tr->tr_size = lsize; list_insert_tail(tr_list, tr); @@ -851,7 +851,7 @@ dsl_dir_tempreserve_space(dsl_dir_t *dd, uint64_t lsize, uint64_t asize, if (err == 0) { struct tempreserve *tr; - tr = kmem_zalloc(sizeof (struct tempreserve), KM_SLEEP); + tr = kmem_zalloc(sizeof (struct tempreserve), KM_PUSHPAGE); tr->tr_dp = dd->dd_pool; tr->tr_size = asize; list_insert_tail(tr_list, tr); diff --git a/module/zfs/dsl_prop.c b/module/zfs/dsl_prop.c index 4b340b63a..64c44b991 100644 --- a/module/zfs/dsl_prop.c +++ b/module/zfs/dsl_prop.c @@ -247,9 +247,9 @@ dsl_prop_register(dsl_dataset_t *ds, const char *propname, return (err); } - cbr = kmem_alloc(sizeof (dsl_prop_cb_record_t), KM_SLEEP); + cbr = kmem_alloc(sizeof (dsl_prop_cb_record_t), KM_PUSHPAGE); cbr->cbr_ds = ds; - cbr->cbr_propname = kmem_alloc(strlen(propname)+1, KM_SLEEP); + cbr->cbr_propname = kmem_alloc(strlen(propname)+1, KM_PUSHPAGE); (void) strcpy((char *)cbr->cbr_propname, propname); cbr->cbr_func = callback; cbr->cbr_arg = cbarg; @@ -534,7 +534,7 @@ dsl_prop_changed_notify(dsl_pool_t *dp, uint64_t ddobj, } mutex_exit(&dd->dd_lock); - za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP); + za = kmem_alloc(sizeof (zap_attribute_t), KM_PUSHPAGE); for (zap_cursor_init(&zc, mos, dd->dd_phys->dd_child_dir_zapobj); zap_cursor_retrieve(&zc, za) == 0; diff --git a/module/zfs/lzjb.c b/module/zfs/lzjb.c index 4da30cf17..43d0df055 100644 --- a/module/zfs/lzjb.c +++ b/module/zfs/lzjb.c @@ -56,7 +56,7 @@ lzjb_compress(void *s_start, void *d_start, size_t s_len, size_t d_len, int n) uint16_t *hp; uint16_t *lempel; - lempel = kmem_zalloc(LEMPEL_SIZE * sizeof (uint16_t), KM_SLEEP); + lempel = kmem_zalloc(LEMPEL_SIZE * sizeof (uint16_t), KM_PUSHPAGE); while (src < (uchar_t *)s_start + s_len) { if ((copymask <<= 1) == (1 << NBBY)) { if (dst >= (uchar_t *)d_start + d_len - 1 - 2 * NBBY) { diff --git a/module/zfs/metaslab.c b/module/zfs/metaslab.c index c33c5e8c7..d06012ffb 100644 --- a/module/zfs/metaslab.c +++ b/module/zfs/metaslab.c @@ -102,7 +102,7 @@ metaslab_class_create(spa_t *spa, space_map_ops_t *ops) { metaslab_class_t *mc; - mc = kmem_zalloc(sizeof (metaslab_class_t), KM_SLEEP); + mc = kmem_zalloc(sizeof (metaslab_class_t), KM_PUSHPAGE); mc->mc_spa = spa; mc->mc_rotor = NULL; @@ -217,7 +217,7 @@ metaslab_group_create(metaslab_class_t *mc, vdev_t *vd) { metaslab_group_t *mg; - mg = kmem_zalloc(sizeof (metaslab_group_t), KM_SLEEP); + mg = kmem_zalloc(sizeof (metaslab_group_t), KM_PUSHPAGE); mutex_init(&mg->mg_lock, NULL, MUTEX_DEFAULT, NULL); avl_create(&mg->mg_metaslab_tree, metaslab_compare, sizeof (metaslab_t), offsetof(struct metaslab, ms_group_node)); @@ -422,9 +422,9 @@ metaslab_pp_load(space_map_t *sm) space_seg_t *ss; ASSERT(sm->sm_ppd == NULL); - sm->sm_ppd = kmem_zalloc(64 * sizeof (uint64_t), KM_SLEEP); + sm->sm_ppd = kmem_zalloc(64 * sizeof (uint64_t), KM_PUSHPAGE); - sm->sm_pp_root = kmem_alloc(sizeof (avl_tree_t), KM_SLEEP); + sm->sm_pp_root = kmem_alloc(sizeof (avl_tree_t), KM_PUSHPAGE); avl_create(sm->sm_pp_root, metaslab_segsize_compare, sizeof (space_seg_t), offsetof(struct space_seg, ss_pp_node)); @@ -725,7 +725,7 @@ metaslab_init(metaslab_group_t *mg, space_map_obj_t *smo, vdev_t *vd = mg->mg_vd; metaslab_t *msp; - msp = kmem_zalloc(sizeof (metaslab_t), KM_SLEEP); + msp = kmem_zalloc(sizeof (metaslab_t), KM_PUSHPAGE); mutex_init(&msp->ms_lock, NULL, MUTEX_DEFAULT, NULL); msp->ms_smo_syncing = *smo; diff --git a/module/zfs/refcount.c b/module/zfs/refcount.c index a61554606..e43807c8e 100644 --- a/module/zfs/refcount.c +++ b/module/zfs/refcount.c @@ -114,7 +114,7 @@ refcount_add_many(refcount_t *rc, uint64_t number, void *holder) int64_t count; if (reference_tracking_enable) { - ref = kmem_cache_alloc(reference_cache, KM_SLEEP); + ref = kmem_cache_alloc(reference_cache, KM_PUSHPAGE); ref->ref_holder = holder; ref->ref_number = number; } @@ -158,7 +158,7 @@ refcount_remove_many(refcount_t *rc, uint64_t number, void *holder) if (reference_history > 0) { ref->ref_removed = kmem_cache_alloc(reference_history_cache, - KM_SLEEP); + KM_PUSHPAGE); list_insert_head(&rc->rc_removed, ref); rc->rc_removed_count++; if (rc->rc_removed_count >= reference_history) { diff --git a/module/zfs/sa.c b/module/zfs/sa.c index a26e5ebdd..d4b28cc90 100644 --- a/module/zfs/sa.c +++ b/module/zfs/sa.c @@ -433,10 +433,10 @@ sa_add_layout_entry(objset_t *os, sa_attr_type_t *attrs, int attr_count, avl_index_t loc; ASSERT(MUTEX_HELD(&sa->sa_lock)); - tb = kmem_zalloc(sizeof (sa_lot_t), KM_SLEEP); + tb = kmem_zalloc(sizeof (sa_lot_t), KM_PUSHPAGE); tb->lot_attr_count = attr_count; tb->lot_attrs = kmem_alloc(sizeof (sa_attr_type_t) * attr_count, - KM_SLEEP); + KM_PUSHPAGE); bcopy(attrs, tb->lot_attrs, sizeof (sa_attr_type_t) * attr_count); tb->lot_num = lot_num; tb->lot_hash = hash; @@ -721,7 +721,7 @@ sa_build_layouts(sa_handle_t *hdl, sa_bulk_attr_t *attr_desc, int attr_count, buf_space = hdl->sa_bonus->db_size - hdrsize; attrs_start = attrs = kmem_alloc(sizeof (sa_attr_type_t) * attr_count, - KM_SLEEP); + KM_PUSHPAGE); lot_count = 0; for (i = 0, len_idx = 0, hash = -1ULL; i != attr_count; i++) { @@ -842,7 +842,7 @@ sa_attr_table_setup(objset_t *os, sa_attr_reg_t *reg_attrs, int count) dmu_objset_type_t ostype = dmu_objset_type(os); sa->sa_user_table = - kmem_zalloc(count * sizeof (sa_attr_type_t), KM_SLEEP); + kmem_zalloc(count * sizeof (sa_attr_type_t), KM_PUSHPAGE); sa->sa_user_table_sz = count * sizeof (sa_attr_type_t); if (sa->sa_reg_attr_obj != 0) { @@ -901,7 +901,7 @@ sa_attr_table_setup(objset_t *os, sa_attr_reg_t *reg_attrs, int count) sa->sa_num_attrs = sa_attr_count; tb = sa->sa_attr_table = - kmem_zalloc(sizeof (sa_attr_table_t) * sa_attr_count, KM_SLEEP); + kmem_zalloc(sizeof (sa_attr_table_t) * sa_attr_count, KM_PUSHPAGE); /* * Attribute table is constructed from requested attribute list, @@ -926,7 +926,7 @@ sa_attr_table_setup(objset_t *os, sa_attr_reg_t *reg_attrs, int count) continue; } tb[ATTR_NUM(value)].sa_name = - kmem_zalloc(strlen(za.za_name) +1, KM_SLEEP); + kmem_zalloc(strlen(za.za_name) +1, KM_PUSHPAGE); (void) strlcpy(tb[ATTR_NUM(value)].sa_name, za.za_name, strlen(za.za_name) +1); } @@ -952,7 +952,7 @@ sa_attr_table_setup(objset_t *os, sa_attr_reg_t *reg_attrs, int count) tb[i].sa_registered = B_FALSE; tb[i].sa_name = kmem_zalloc(strlen(sa_legacy_attrs[i].sa_name) +1, - KM_SLEEP); + KM_PUSHPAGE); (void) strlcpy(tb[i].sa_name, sa_legacy_attrs[i].sa_name, strlen(sa_legacy_attrs[i].sa_name) + 1); @@ -970,7 +970,7 @@ sa_attr_table_setup(objset_t *os, sa_attr_reg_t *reg_attrs, int count) tb[attr_id].sa_byteswap = reg_attrs[i].sa_byteswap; tb[attr_id].sa_attr = attr_id; tb[attr_id].sa_name = - kmem_zalloc(strlen(reg_attrs[i].sa_name) + 1, KM_SLEEP); + kmem_zalloc(strlen(reg_attrs[i].sa_name) + 1, KM_PUSHPAGE); (void) strlcpy(tb[attr_id].sa_name, reg_attrs[i].sa_name, strlen(reg_attrs[i].sa_name) + 1); } @@ -1007,7 +1007,7 @@ sa_setup(objset_t *os, uint64_t sa_obj, sa_attr_reg_t *reg_attrs, int count, return (0); } - sa = kmem_zalloc(sizeof (sa_os_t), KM_SLEEP); + sa = kmem_zalloc(sizeof (sa_os_t), KM_PUSHPAGE); mutex_init(&sa->sa_lock, NULL, MUTEX_DEFAULT, NULL); sa->sa_master_obj = sa_obj; @@ -1055,7 +1055,7 @@ sa_setup(objset_t *os, uint64_t sa_obj, sa_attr_reg_t *reg_attrs, int count, uint64_t lot_num; lot_attrs = kmem_zalloc(sizeof (sa_attr_type_t) * - za.za_num_integers, KM_SLEEP); + za.za_num_integers, KM_PUSHPAGE); if ((error = (zap_lookup(os, sa->sa_layout_attr_obj, za.za_name, 2, za.za_num_integers, @@ -1540,14 +1540,14 @@ sa_find_idx_tab(objset_t *os, dmu_object_type_t bonustype, void *data) } /* No such luck, create a new entry */ - idx_tab = kmem_zalloc(sizeof (sa_idx_tab_t), KM_SLEEP); + idx_tab = kmem_zalloc(sizeof (sa_idx_tab_t), KM_PUSHPAGE); idx_tab->sa_idx_tab = - kmem_zalloc(sizeof (uint32_t) * sa->sa_num_attrs, KM_SLEEP); + kmem_zalloc(sizeof (uint32_t) * sa->sa_num_attrs, KM_PUSHPAGE); idx_tab->sa_layout = tb; refcount_create(&idx_tab->sa_refcount); if (tb->lot_var_sizes) idx_tab->sa_variable_lengths = kmem_alloc(sizeof (uint16_t) * - tb->lot_var_sizes, KM_SLEEP); + tb->lot_var_sizes, KM_PUSHPAGE); sa_attr_iter(os, hdr, bonustype, sa_build_idx_tab, tb, idx_tab); diff --git a/module/zfs/spa.c b/module/zfs/spa.c index 7abe69902..a3d52c8b1 100644 --- a/module/zfs/spa.c +++ b/module/zfs/spa.c @@ -150,7 +150,7 @@ spa_prop_add_list(nvlist_t *nvl, zpool_prop_t prop, char *strval, const char *propname = zpool_prop_to_name(prop); nvlist_t *propval; - VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&propval, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_uint64(propval, ZPROP_SOURCE, src) == 0); if (strval != NULL) @@ -237,7 +237,7 @@ spa_prop_get(spa_t *spa, nvlist_t **nvp) zap_attribute_t za; int err; - err = nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_SLEEP); + err = nvlist_alloc(nvp, NV_UNIQUE_NAME, KM_PUSHPAGE); if (err) return err; @@ -289,7 +289,7 @@ spa_prop_get(spa_t *spa, nvlist_t **nvp) strval = kmem_alloc( MAXNAMELEN + strlen(MOS_DIR_NAME) + 1, - KM_SLEEP); + KM_PUSHPAGE); dsl_dataset_name(ds, strval); dsl_dataset_rele(ds, FTAG); rw_exit(&dp->dp_config_rwlock); @@ -308,7 +308,7 @@ spa_prop_get(spa_t *spa, nvlist_t **nvp) case 1: /* string property */ - strval = kmem_alloc(za.za_num_integers, KM_SLEEP); + strval = kmem_alloc(za.za_num_integers, KM_PUSHPAGE); err = zap_lookup(mos, spa->spa_pool_props_object, za.za_name, 1, za.za_num_integers, strval); if (err) { @@ -528,7 +528,7 @@ spa_configfile_set(spa_t *spa, nvlist_t *nvp, boolean_t need_sync) return; dp = kmem_alloc(sizeof (spa_config_dirent_t), - KM_SLEEP); + KM_PUSHPAGE); if (cachefile[0] == '\0') dp->scd_path = spa_strdup(spa_config_path); @@ -1140,7 +1140,7 @@ spa_load_spares(spa_t *spa) * active configuration, then we also mark this vdev as an active spare. */ spa->spa_spares.sav_vdevs = kmem_alloc(nspares * sizeof (void *), - KM_SLEEP); + KM_PUSHPAGE); for (i = 0; i < spa->spa_spares.sav_count; i++) { VERIFY(spa_config_parse(spa, &vd, spares[i], NULL, 0, VDEV_ALLOC_SPARE) == 0); @@ -1188,7 +1188,7 @@ spa_load_spares(spa_t *spa) DATA_TYPE_NVLIST_ARRAY) == 0); spares = kmem_alloc(spa->spa_spares.sav_count * sizeof (void *), - KM_SLEEP); + KM_PUSHPAGE); for (i = 0; i < spa->spa_spares.sav_count; i++) spares[i] = vdev_config_generate(spa, spa->spa_spares.sav_vdevs[i], B_TRUE, VDEV_CONFIG_SPARE); @@ -1222,7 +1222,7 @@ spa_load_l2cache(spa_t *spa) if (sav->sav_config != NULL) { VERIFY(nvlist_lookup_nvlist_array(sav->sav_config, ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0); - newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_SLEEP); + newvdevs = kmem_alloc(nl2cache * sizeof (void *), KM_PUSHPAGE); } else { nl2cache = 0; } @@ -1316,7 +1316,7 @@ spa_load_l2cache(spa_t *spa) VERIFY(nvlist_remove(sav->sav_config, ZPOOL_CONFIG_L2CACHE, DATA_TYPE_NVLIST_ARRAY) == 0); - l2cache = kmem_alloc(sav->sav_count * sizeof (void *), KM_SLEEP); + l2cache = kmem_alloc(sav->sav_count * sizeof (void *), KM_PUSHPAGE); for (i = 0; i < sav->sav_count; i++) l2cache[i] = vdev_config_generate(spa, sav->sav_vdevs[i], B_TRUE, VDEV_CONFIG_L2CACHE); @@ -1342,7 +1342,7 @@ load_nvlist(spa_t *spa, uint64_t obj, nvlist_t **value) nvsize = *(uint64_t *)db->db_data; dmu_buf_rele(db, FTAG); - packed = kmem_alloc(nvsize, KM_SLEEP | KM_NODEBUG); + packed = kmem_alloc(nvsize, KM_PUSHPAGE | KM_NODEBUG); error = dmu_read(spa->spa_meta_objset, obj, 0, nvsize, packed, DMU_READ_PREFETCH); if (error == 0) @@ -1398,8 +1398,8 @@ spa_config_valid(spa_t *spa, nvlist_t *config) uint64_t idx = 0; child = kmem_alloc(rvd->vdev_children * sizeof (nvlist_t **), - KM_SLEEP); - VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_SLEEP) == 0); + KM_PUSHPAGE); + VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); for (c = 0; c < rvd->vdev_children; c++) { vdev_t *tvd = rvd->vdev_child[c]; @@ -1754,7 +1754,7 @@ spa_try_repair(spa_t *spa, nvlist_t *config) &glist, &gcount) != 0) return; - vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_SLEEP); + vd = kmem_zalloc(gcount * sizeof (vdev_t *), KM_PUSHPAGE); /* attempt to online all the vdevs & validate */ attempt_reopen = B_TRUE; @@ -1840,7 +1840,7 @@ spa_load(spa_t *spa, spa_load_state_t state, spa_import_type_t type, if (nvlist_lookup_nvlist(config, ZPOOL_CONFIG_SPLIT, &nvl) == 0) { VERIFY(nvlist_dup(nvl, &spa->spa_config_splitting, - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); } gethrestime(&spa->spa_loaded_ts); @@ -2497,7 +2497,7 @@ spa_open_common(const char *pool, spa_t **spapp, void *tag, nvlist_t *nvpolicy, */ if (config != NULL && spa->spa_config) { VERIFY(nvlist_dup(spa->spa_config, config, - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); VERIFY(nvlist_add_nvlist(*config, ZPOOL_CONFIG_LOAD_INFO, spa->spa_load_info) == 0); @@ -2873,13 +2873,13 @@ spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs, &olddevs, &oldndevs) == 0); newdevs = kmem_alloc(sizeof (void *) * - (ndevs + oldndevs), KM_SLEEP); + (ndevs + oldndevs), KM_PUSHPAGE); for (i = 0; i < oldndevs; i++) VERIFY(nvlist_dup(olddevs[i], &newdevs[i], - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); for (i = 0; i < ndevs; i++) VERIFY(nvlist_dup(devs[i], &newdevs[i + oldndevs], - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); VERIFY(nvlist_remove(sav->sav_config, config, DATA_TYPE_NVLIST_ARRAY) == 0); @@ -2894,7 +2894,7 @@ spa_set_aux_vdevs(spa_aux_vdev_t *sav, nvlist_t **devs, int ndevs, * Generate a new dev list. */ VERIFY(nvlist_alloc(&sav->sav_config, NV_UNIQUE_NAME, - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); VERIFY(nvlist_add_nvlist_array(sav->sav_config, config, devs, ndevs) == 0); } @@ -3020,7 +3020,7 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_SPARES, &spares, &nspares) == 0) { VERIFY(nvlist_alloc(&spa->spa_spares.sav_config, NV_UNIQUE_NAME, - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES, spares, nspares) == 0); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); @@ -3035,7 +3035,7 @@ spa_create(const char *pool, nvlist_t *nvroot, nvlist_t *props, if (nvlist_lookup_nvlist_array(nvroot, ZPOOL_CONFIG_L2CACHE, &l2cache, &nl2cache) == 0) { VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config, - NV_UNIQUE_NAME, KM_SLEEP) == 0); + NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config, ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); @@ -3173,7 +3173,7 @@ spa_generate_rootconf(char *devpath, char *devid, uint64_t *guid) /* * Put this pool's top-level vdevs into a root vdev. */ - VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_string(nvroot, ZPOOL_CONFIG_TYPE, VDEV_TYPE_ROOT) == 0); VERIFY(nvlist_add_uint64(nvroot, ZPOOL_CONFIG_ID, 0ULL) == 0); @@ -3484,7 +3484,7 @@ spa_import(const char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags) ZPOOL_CONFIG_SPARES, DATA_TYPE_NVLIST_ARRAY) == 0); else VERIFY(nvlist_alloc(&spa->spa_spares.sav_config, - NV_UNIQUE_NAME, KM_SLEEP) == 0); + NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_nvlist_array(spa->spa_spares.sav_config, ZPOOL_CONFIG_SPARES, spares, nspares) == 0); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); @@ -3499,7 +3499,7 @@ spa_import(const char *pool, nvlist_t *config, nvlist_t *props, uint64_t flags) ZPOOL_CONFIG_L2CACHE, DATA_TYPE_NVLIST_ARRAY) == 0); else VERIFY(nvlist_alloc(&spa->spa_l2cache.sav_config, - NV_UNIQUE_NAME, KM_SLEEP) == 0); + NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_nvlist_array(spa->spa_l2cache.sav_config, ZPOOL_CONFIG_L2CACHE, l2cache, nl2cache) == 0); spa_config_enter(spa, SCL_ALL, FTAG, RW_WRITER); @@ -3582,7 +3582,7 @@ spa_tryimport(nvlist_t *tryconfig) * pools are bootable. */ if ((!error || error == EEXIST) && spa->spa_bootfs) { - char *tmpname = kmem_alloc(MAXPATHLEN, KM_SLEEP); + char *tmpname = kmem_alloc(MAXPATHLEN, KM_PUSHPAGE); /* * We have to play games with the name since the @@ -3591,7 +3591,7 @@ spa_tryimport(nvlist_t *tryconfig) if (dsl_dsobj_to_dsname(spa_name(spa), spa->spa_bootfs, tmpname) == 0) { char *cp; - char *dsname = kmem_alloc(MAXPATHLEN, KM_SLEEP); + char *dsname = kmem_alloc(MAXPATHLEN, KM_PUSHPAGE); cp = strchr(tmpname, '/'); if (cp == NULL) { @@ -3996,7 +3996,7 @@ spa_vdev_attach(spa_t *spa, uint64_t guid, nvlist_t *nvroot, int replacing) if (strcmp(oldvd->vdev_path, newvd->vdev_path) == 0) { spa_strfree(oldvd->vdev_path); oldvd->vdev_path = kmem_alloc(strlen(newvd->vdev_path) + 5, - KM_SLEEP); + KM_PUSHPAGE); (void) sprintf(oldvd->vdev_path, "%s/%s", newvd->vdev_path, "old"); if (oldvd->vdev_devid != NULL) { @@ -4391,8 +4391,8 @@ spa_vdev_split_mirror(spa_t *spa, char *newname, nvlist_t *config, nvlist_lookup_nvlist(nvl, ZPOOL_CONFIG_L2CACHE, &tmp) == 0) return (spa_vdev_exit(spa, NULL, txg, EINVAL)); - vml = kmem_zalloc(children * sizeof (vdev_t *), KM_SLEEP); - glist = kmem_zalloc(children * sizeof (uint64_t), KM_SLEEP); + vml = kmem_zalloc(children * sizeof (vdev_t *), KM_PUSHPAGE); + glist = kmem_zalloc(children * sizeof (uint64_t), KM_PUSHPAGE); /* then, loop over each vdev and validate it */ for (c = 0; c < children; c++) { @@ -4472,7 +4472,7 @@ spa_vdev_split_mirror(spa_t *spa, char *newname, nvlist_t *config, * Temporarily record the splitting vdevs in the spa config. This * will disappear once the config is regenerated. */ - VERIFY(nvlist_alloc(&nvl, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&nvl, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_uint64_array(nvl, ZPOOL_CONFIG_SPLIT_LIST, glist, children) == 0); kmem_free(glist, children * sizeof (uint64_t)); @@ -4519,7 +4519,7 @@ spa_vdev_split_mirror(spa_t *spa, char *newname, nvlist_t *config, /* if that worked, generate a real config for the new pool */ if (newspa->spa_root_vdev != NULL) { VERIFY(nvlist_alloc(&newspa->spa_config_splitting, - NV_UNIQUE_NAME, KM_SLEEP) == 0); + NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_uint64(newspa->spa_config_splitting, ZPOOL_CONFIG_SPLIT_GUID, spa_guid(spa)) == 0); spa_config_set(newspa, spa_config_generate(newspa, NULL, -1ULL, @@ -4631,12 +4631,12 @@ spa_vdev_remove_aux(nvlist_t *config, char *name, nvlist_t **dev, int count, int i, j; if (count > 1) - newdev = kmem_alloc((count - 1) * sizeof (void *), KM_SLEEP); + newdev = kmem_alloc((count - 1) * sizeof (void *), KM_PUSHPAGE); for (i = 0, j = 0; i < count; i++) { if (dev[i] == dev_to_remove) continue; - VERIFY(nvlist_dup(dev[i], &newdev[j++], KM_SLEEP) == 0); + VERIFY(nvlist_dup(dev[i], &newdev[j++], KM_PUSHPAGE) == 0); } VERIFY(nvlist_remove(config, name, DATA_TYPE_NVLIST_ARRAY) == 0); @@ -5291,10 +5291,10 @@ spa_sync_nvlist(spa_t *spa, uint64_t obj, nvlist_t *nv, dmu_tx_t *tx) * saves us a pre-read to get data we don't actually care about. */ bufsize = P2ROUNDUP(nvsize, SPA_CONFIG_BLOCKSIZE); - packed = vmem_alloc(bufsize, KM_SLEEP); + packed = vmem_alloc(bufsize, KM_PUSHPAGE); VERIFY(nvlist_pack(nv, &packed, &nvsize, NV_ENCODE_XDR, - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); bzero(packed + nvsize, bufsize - nvsize); dmu_write(spa->spa_meta_objset, obj, 0, bufsize, packed, tx); @@ -5332,11 +5332,11 @@ spa_sync_aux_dev(spa_t *spa, spa_aux_vdev_t *sav, dmu_tx_t *tx, &sav->sav_object, tx) == 0); } - VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&nvroot, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); if (sav->sav_count == 0) { VERIFY(nvlist_add_nvlist_array(nvroot, config, NULL, 0) == 0); } else { - list = kmem_alloc(sav->sav_count * sizeof (void *), KM_SLEEP); + list = kmem_alloc(sav->sav_count * sizeof (void *), KM_PUSHPAGE); for (i = 0; i < sav->sav_count; i++) list[i] = vdev_config_generate(spa, sav->sav_vdevs[i], B_FALSE, VDEV_CONFIG_L2CACHE); diff --git a/module/zfs/spa_config.c b/module/zfs/spa_config.c index d814ae217..c86884148 100644 --- a/module/zfs/spa_config.c +++ b/module/zfs/spa_config.c @@ -83,7 +83,7 @@ spa_config_load(void) /* * Open the configuration file. */ - pathname = kmem_alloc(MAXPATHLEN, KM_SLEEP); + pathname = kmem_alloc(MAXPATHLEN, KM_PUSHPAGE); (void) snprintf(pathname, MAXPATHLEN, "%s%s", (rootdir != NULL) ? "./" : "", spa_config_path); @@ -98,7 +98,7 @@ spa_config_load(void) if (kobj_get_filesize(file, &fsize) != 0) goto out; - buf = kmem_alloc(fsize, KM_SLEEP | KM_NODEBUG); + buf = kmem_alloc(fsize, KM_PUSHPAGE | KM_NODEBUG); /* * Read the nvlist from the file. @@ -109,7 +109,7 @@ spa_config_load(void) /* * Unpack the nvlist. */ - if (nvlist_unpack(buf, fsize, &nvlist, KM_SLEEP) != 0) + if (nvlist_unpack(buf, fsize, &nvlist, KM_PUSHPAGE) != 0) goto out; /* @@ -161,11 +161,11 @@ spa_config_write(spa_config_dirent_t *dp, nvlist_t *nvl) */ VERIFY(nvlist_size(nvl, &buflen, NV_ENCODE_XDR) == 0); - buf = kmem_alloc(buflen, KM_SLEEP | KM_NODEBUG); - temp = kmem_zalloc(MAXPATHLEN, KM_SLEEP); + buf = kmem_alloc(buflen, KM_PUSHPAGE | KM_NODEBUG); + temp = kmem_zalloc(MAXPATHLEN, KM_PUSHPAGE); VERIFY(nvlist_pack(nvl, &buf, &buflen, NV_ENCODE_XDR, - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); /* * Write the configuration to disk. We need to do the traditional @@ -234,7 +234,7 @@ spa_config_sync(spa_t *target, boolean_t removing, boolean_t postsysevent) if (nvl == NULL) VERIFY(nvlist_alloc(&nvl, NV_UNIQUE_NAME, - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); VERIFY(nvlist_add_nvlist(nvl, spa->spa_name, spa->spa_config) == 0); @@ -277,7 +277,7 @@ spa_all_configs(uint64_t *generation) if (*generation == spa_config_generation) return (NULL); - VERIFY(nvlist_alloc(&pools, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&pools, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); mutex_enter(&spa_namespace_lock); while ((spa = spa_next(spa)) != NULL) { @@ -334,7 +334,7 @@ spa_config_generate(spa_t *spa, vdev_t *vd, uint64_t txg, int getstats) if (txg == -1ULL) txg = spa->spa_config_txg; - VERIFY(nvlist_alloc(&config, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&config, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_uint64(config, ZPOOL_CONFIG_VERSION, spa_version(spa)) == 0); @@ -413,21 +413,21 @@ spa_config_generate(spa_t *spa, vdev_t *vd, uint64_t txg, int getstats) ddt_stat_t *dds; ddt_object_t *ddo; - ddh = kmem_zalloc(sizeof (ddt_histogram_t), KM_SLEEP); + ddh = kmem_zalloc(sizeof (ddt_histogram_t), KM_PUSHPAGE); ddt_get_dedup_histogram(spa, ddh); VERIFY(nvlist_add_uint64_array(config, ZPOOL_CONFIG_DDT_HISTOGRAM, (uint64_t *)ddh, sizeof (*ddh) / sizeof (uint64_t)) == 0); kmem_free(ddh, sizeof (ddt_histogram_t)); - ddo = kmem_zalloc(sizeof (ddt_object_t), KM_SLEEP); + ddo = kmem_zalloc(sizeof (ddt_object_t), KM_PUSHPAGE); ddt_get_dedup_object_stats(spa, ddo); VERIFY(nvlist_add_uint64_array(config, ZPOOL_CONFIG_DDT_OBJ_STATS, (uint64_t *)ddo, sizeof (*ddo) / sizeof (uint64_t)) == 0); kmem_free(ddo, sizeof (ddt_object_t)); - dds = kmem_zalloc(sizeof (ddt_stat_t), KM_SLEEP); + dds = kmem_zalloc(sizeof (ddt_stat_t), KM_PUSHPAGE); ddt_get_dedup_stats(spa, dds); VERIFY(nvlist_add_uint64_array(config, ZPOOL_CONFIG_DDT_STATS, diff --git a/module/zfs/spa_history.c b/module/zfs/spa_history.c index 7a2537875..9fb75f391 100644 --- a/module/zfs/spa_history.c +++ b/module/zfs/spa_history.c @@ -234,7 +234,7 @@ spa_history_log_sync(void *arg1, void *arg2, dmu_tx_t *tx) } #endif - VERIFY(nvlist_alloc(&nvrecord, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&nvrecord, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_uint64(nvrecord, ZPOOL_HIST_TIME, gethrestime_sec()) == 0); VERIFY(nvlist_add_uint64(nvrecord, ZPOOL_HIST_WHO, hap->ha_uid) == 0); @@ -266,10 +266,10 @@ spa_history_log_sync(void *arg1, void *arg2, dmu_tx_t *tx) } VERIFY(nvlist_size(nvrecord, &reclen, NV_ENCODE_XDR) == 0); - record_packed = kmem_alloc(reclen, KM_SLEEP); + record_packed = kmem_alloc(reclen, KM_PUSHPAGE); VERIFY(nvlist_pack(nvrecord, &record_packed, &reclen, - NV_ENCODE_XDR, KM_SLEEP) == 0); + NV_ENCODE_XDR, KM_PUSHPAGE) == 0); mutex_enter(&spa->spa_history_lock); if (hap->ha_log_type == LOG_CMD_POOL_CREATE) @@ -316,7 +316,7 @@ spa_history_log(spa_t *spa, const char *history_str, history_log_type_t what) return (err); } - ha = kmem_alloc(sizeof (history_arg_t), KM_SLEEP); + ha = kmem_alloc(sizeof (history_arg_t), KM_PUSHPAGE); ha->ha_history_str = strdup(history_str); ha->ha_zone = strdup(spa_history_zone()); ha->ha_log_type = what; @@ -442,7 +442,7 @@ log_internal(history_internal_events_t event, spa_t *spa, if (tx->tx_txg == TXG_INITIAL) return; - ha = kmem_alloc(sizeof (history_arg_t), KM_SLEEP); + ha = kmem_alloc(sizeof (history_arg_t), KM_PUSHPAGE); va_copy(adx_copy, adx); ha->ha_history_str = kmem_vasprintf(fmt, adx_copy); va_end(adx_copy); diff --git a/module/zfs/spa_misc.c b/module/zfs/spa_misc.c index c82dca6c5..4a8e6adfd 100644 --- a/module/zfs/spa_misc.c +++ b/module/zfs/spa_misc.c @@ -425,7 +425,7 @@ spa_add(const char *name, nvlist_t *config, const char *altroot) ASSERT(MUTEX_HELD(&spa_namespace_lock)); - spa = kmem_zalloc(sizeof (spa_t), KM_SLEEP | KM_NODEBUG); + spa = kmem_zalloc(sizeof (spa_t), KM_PUSHPAGE | KM_NODEBUG); mutex_init(&spa->spa_async_lock, NULL, MUTEX_DEFAULT, NULL); mutex_init(&spa->spa_errlist_lock, NULL, MUTEX_DEFAULT, NULL); @@ -472,12 +472,12 @@ spa_add(const char *name, nvlist_t *config, const char *altroot) list_create(&spa->spa_config_list, sizeof (spa_config_dirent_t), offsetof(spa_config_dirent_t, scd_link)); - dp = kmem_zalloc(sizeof (spa_config_dirent_t), KM_SLEEP); + dp = kmem_zalloc(sizeof (spa_config_dirent_t), KM_PUSHPAGE); dp->scd_path = altroot ? NULL : spa_strdup(spa_config_path); list_insert_head(&spa->spa_config_list, dp); VERIFY(nvlist_alloc(&spa->spa_load_info, NV_UNIQUE_NAME, - KM_SLEEP) == 0); + KM_PUSHPAGE) == 0); if (config != NULL) VERIFY(nvlist_dup(config, &spa->spa_config, 0) == 0); @@ -647,7 +647,7 @@ spa_aux_add(vdev_t *vd, avl_tree_t *avl) if ((aux = avl_find(avl, &search, &where)) != NULL) { aux->aux_count++; } else { - aux = kmem_zalloc(sizeof (spa_aux_t), KM_SLEEP); + aux = kmem_zalloc(sizeof (spa_aux_t), KM_PUSHPAGE); aux->aux_guid = vd->vdev_guid; aux->aux_count = 1; avl_insert(avl, aux, where); @@ -1131,7 +1131,7 @@ spa_strdup(const char *s) char *new; len = strlen(s); - new = kmem_alloc(len + 1, KM_SLEEP); + new = kmem_alloc(len + 1, KM_PUSHPAGE); bcopy(s, new, len); new[len] = '\0'; diff --git a/module/zfs/space_map.c b/module/zfs/space_map.c index 1ce7b2a3d..9c0cdb6be 100644 --- a/module/zfs/space_map.c +++ b/module/zfs/space_map.c @@ -134,7 +134,7 @@ space_map_add(space_map_t *sm, uint64_t start, uint64_t size) avl_remove(sm->sm_pp_root, ss_after); ss = ss_after; } else { - ss = kmem_alloc(sizeof (*ss), KM_SLEEP); + ss = kmem_alloc(sizeof (*ss), KM_PUSHPAGE); ss->ss_start = start; ss->ss_end = end; avl_insert(&sm->sm_root, ss, where); @@ -181,7 +181,7 @@ space_map_remove(space_map_t *sm, uint64_t start, uint64_t size) avl_remove(sm->sm_pp_root, ss); if (left_over && right_over) { - newseg = kmem_alloc(sizeof (*newseg), KM_SLEEP); + newseg = kmem_alloc(sizeof (*newseg), KM_PUSHPAGE); newseg->ss_start = end; newseg->ss_end = ss->ss_end; ss->ss_end = start; @@ -551,7 +551,7 @@ space_map_ref_add_node(avl_tree_t *t, uint64_t offset, int64_t refcnt) { space_ref_t *sr; - sr = kmem_alloc(sizeof (*sr), KM_SLEEP); + sr = kmem_alloc(sizeof (*sr), KM_PUSHPAGE); sr->sr_offset = offset; sr->sr_refcnt = refcnt; diff --git a/module/zfs/txg.c b/module/zfs/txg.c index aefda6f69..e14bff269 100644 --- a/module/zfs/txg.c +++ b/module/zfs/txg.c @@ -349,7 +349,7 @@ txg_dispatch_callbacks(dsl_pool_t *dp, uint64_t txg) TASKQ_THREADS_CPU_PCT | TASKQ_PREPOPULATE); } - cb_list = kmem_alloc(sizeof (list_t), KM_SLEEP); + cb_list = kmem_alloc(sizeof (list_t), KM_PUSHPAGE); list_create(cb_list, sizeof (dmu_tx_callback_t), offsetof(dmu_tx_callback_t, dcb_node)); diff --git a/module/zfs/unique.c b/module/zfs/unique.c index 8c1d2e2f9..5c7ca4875 100644 --- a/module/zfs/unique.c +++ b/module/zfs/unique.c @@ -79,7 +79,7 @@ uint64_t unique_insert(uint64_t value) { avl_index_t idx; - unique_t *un = kmem_alloc(sizeof (unique_t), KM_SLEEP); + unique_t *un = kmem_alloc(sizeof (unique_t), KM_PUSHPAGE); un->un_value = value; diff --git a/module/zfs/vdev.c b/module/zfs/vdev.c index 0c8ce1b1c..06c7d0c89 100644 --- a/module/zfs/vdev.c +++ b/module/zfs/vdev.c @@ -195,7 +195,7 @@ vdev_add_child(vdev_t *pvd, vdev_t *cvd) pvd->vdev_children = MAX(pvd->vdev_children, id + 1); newsize = pvd->vdev_children * sizeof (vdev_t *); - newchild = kmem_zalloc(newsize, KM_SLEEP); + newchild = kmem_zalloc(newsize, KM_PUSHPAGE); if (pvd->vdev_child != NULL) { bcopy(pvd->vdev_child, newchild, oldsize); kmem_free(pvd->vdev_child, oldsize); @@ -265,7 +265,7 @@ vdev_compact_children(vdev_t *pvd) if (pvd->vdev_child[c]) newc++; - newchild = kmem_alloc(newc * sizeof (vdev_t *), KM_SLEEP); + newchild = kmem_alloc(newc * sizeof (vdev_t *), KM_PUSHPAGE); for (c = newc = 0; c < oldc; c++) { if ((cvd = pvd->vdev_child[c]) != NULL) { @@ -288,7 +288,7 @@ vdev_alloc_common(spa_t *spa, uint_t id, uint64_t guid, vdev_ops_t *ops) vdev_t *vd; int t; - vd = kmem_zalloc(sizeof (vdev_t), KM_SLEEP); + vd = kmem_zalloc(sizeof (vdev_t), KM_PUSHPAGE); if (spa->spa_root_vdev == NULL) { ASSERT(ops == &vdev_root_ops); @@ -838,7 +838,7 @@ vdev_metaslab_init(vdev_t *vd, uint64_t txg) ASSERT(oldc <= newc); - mspp = kmem_zalloc(newc * sizeof (*mspp), KM_SLEEP | KM_NODEBUG); + mspp = kmem_zalloc(newc * sizeof (*mspp), KM_PUSHPAGE | KM_NODEBUG); if (oldc != 0) { bcopy(vd->vdev_ms, mspp, oldc * sizeof (*mspp)); @@ -993,7 +993,7 @@ vdev_probe(vdev_t *vd, zio_t *zio) mutex_enter(&vd->vdev_probe_lock); if ((pio = vd->vdev_probe_zio) == NULL) { - vps = kmem_zalloc(sizeof (*vps), KM_SLEEP); + vps = kmem_zalloc(sizeof (*vps), KM_PUSHPAGE); vps->vps_flags = ZIO_FLAG_CANFAIL | ZIO_FLAG_PROBE | ZIO_FLAG_DONT_CACHE | ZIO_FLAG_DONT_AGGREGATE | diff --git a/module/zfs/vdev_cache.c b/module/zfs/vdev_cache.c index e2f8040d1..bf4ae7b2b 100644 --- a/module/zfs/vdev_cache.c +++ b/module/zfs/vdev_cache.c @@ -177,7 +177,7 @@ vdev_cache_allocate(zio_t *zio) vdev_cache_evict(vc, ve); } - ve = kmem_zalloc(sizeof (vdev_cache_entry_t), KM_SLEEP); + ve = kmem_zalloc(sizeof (vdev_cache_entry_t), KM_PUSHPAGE); ve->ve_offset = offset; ve->ve_lastused = ddi_get_lbolt(); ve->ve_data = zio_buf_alloc(VCBS); @@ -274,7 +274,7 @@ vdev_cache_read(zio_t *zio) mutex_enter(&vc->vc_lock); - ve_search = kmem_alloc(sizeof(vdev_cache_entry_t), KM_SLEEP); + ve_search = kmem_alloc(sizeof(vdev_cache_entry_t), KM_PUSHPAGE); ve_search->ve_offset = cache_offset; ve = avl_find(&vc->vc_offset_tree, ve_search, NULL); kmem_free(ve_search, sizeof(vdev_cache_entry_t)); diff --git a/module/zfs/vdev_disk.c b/module/zfs/vdev_disk.c index eee03d080..fd40b1005 100644 --- a/module/zfs/vdev_disk.c +++ b/module/zfs/vdev_disk.c @@ -236,7 +236,7 @@ vdev_disk_open(vdev_t *v, uint64_t *psize, uint64_t *ashift) return EINVAL; } - vd = kmem_zalloc(sizeof(vdev_disk_t), KM_SLEEP); + vd = kmem_zalloc(sizeof(vdev_disk_t), KM_PUSHPAGE); if (vd == NULL) return ENOMEM; @@ -320,7 +320,7 @@ vdev_disk_dio_alloc(int bio_count) int i; dr = kmem_zalloc(sizeof(dio_request_t) + - sizeof(struct bio *) * bio_count, KM_SLEEP); + sizeof(struct bio *) * bio_count, KM_PUSHPAGE); if (dr) { init_completion(&dr->dr_comp); atomic_set(&dr->dr_ref, 0); @@ -789,7 +789,7 @@ vdev_disk_read_rootlabel(char *devpath, char *devid, nvlist_t **config) } size = P2ALIGN_TYPED(s, sizeof(vdev_label_t), uint64_t); - label = vmem_alloc(sizeof(vdev_label_t), KM_SLEEP); + label = vmem_alloc(sizeof(vdev_label_t), KM_PUSHPAGE); for (i = 0; i < VDEV_LABELS; i++) { uint64_t offset, state, txg = 0; diff --git a/module/zfs/vdev_file.c b/module/zfs/vdev_file.c index ce49fe08f..25d0bad3f 100644 --- a/module/zfs/vdev_file.c +++ b/module/zfs/vdev_file.c @@ -72,7 +72,7 @@ vdev_file_open(vdev_t *vd, uint64_t *psize, uint64_t *ashift) goto skip_open; } - vf = vd->vdev_tsd = kmem_zalloc(sizeof (vdev_file_t), KM_SLEEP); + vf = vd->vdev_tsd = kmem_zalloc(sizeof (vdev_file_t), KM_PUSHPAGE); /* * We always open the files from the root of the global zone, even if diff --git a/module/zfs/vdev_label.c b/module/zfs/vdev_label.c index 3774d7152..7ac23500f 100644 --- a/module/zfs/vdev_label.c +++ b/module/zfs/vdev_label.c @@ -212,7 +212,7 @@ vdev_config_generate(spa_t *spa, vdev_t *vd, boolean_t getstats, { nvlist_t *nv = NULL; - VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&nv, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_string(nv, ZPOOL_CONFIG_TYPE, vd->vdev_ops->vdev_op_type) == 0); @@ -319,7 +319,7 @@ vdev_config_generate(spa_t *spa, vdev_t *vd, boolean_t getstats, ASSERT(!vd->vdev_ishole); child = kmem_alloc(vd->vdev_children * sizeof (nvlist_t *), - KM_SLEEP); + KM_PUSHPAGE); for (c = 0, idx = 0; c < vd->vdev_children; c++) { vdev_t *cvd = vd->vdev_child[c]; @@ -408,7 +408,7 @@ vdev_top_config_generate(spa_t *spa, nvlist_t *config) uint64_t *array; uint_t c, idx; - array = kmem_alloc(rvd->vdev_children * sizeof (uint64_t), KM_SLEEP); + array = kmem_alloc(rvd->vdev_children * sizeof (uint64_t), KM_PUSHPAGE); for (c = 0, idx = 0; c < rvd->vdev_children; c++) { vdev_t *tvd = rvd->vdev_child[c]; @@ -709,7 +709,7 @@ vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason) * active hot spare (in which case we want to revert the * labels). */ - VERIFY(nvlist_alloc(&label, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&label, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_VERSION, spa_version(spa)) == 0); @@ -722,7 +722,7 @@ vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason) /* * For level 2 ARC devices, add a special label. */ - VERIFY(nvlist_alloc(&label, NV_UNIQUE_NAME, KM_SLEEP) == 0); + VERIFY(nvlist_alloc(&label, NV_UNIQUE_NAME, KM_PUSHPAGE) == 0); VERIFY(nvlist_add_uint64(label, ZPOOL_CONFIG_VERSION, spa_version(spa)) == 0); @@ -749,7 +749,7 @@ vdev_label_init(vdev_t *vd, uint64_t crtxg, vdev_labeltype_t reason) buf = vp->vp_nvlist; buflen = sizeof (vp->vp_nvlist); - error = nvlist_pack(label, &buf, &buflen, NV_ENCODE_XDR, KM_SLEEP); + error = nvlist_pack(label, &buf, &buflen, NV_ENCODE_XDR, KM_PUSHPAGE); if (error != 0) { nvlist_free(label); zio_buf_free(vp, sizeof (vdev_phys_t)); @@ -1061,7 +1061,7 @@ vdev_label_sync(zio_t *zio, vdev_t *vd, int l, uint64_t txg, int flags) buf = vp->vp_nvlist; buflen = sizeof (vp->vp_nvlist); - if (nvlist_pack(label, &buf, &buflen, NV_ENCODE_XDR, KM_SLEEP) == 0) { + if (nvlist_pack(label, &buf, &buflen, NV_ENCODE_XDR, KM_PUSHPAGE) == 0) { for (; l < VDEV_LABELS; l += 2) { vdev_label_write(zio, vd, l, vp, offsetof(vdev_label_t, vl_vdev_phys), @@ -1094,7 +1094,7 @@ vdev_label_sync_list(spa_t *spa, int l, uint64_t txg, int flags) ASSERT(!vd->vdev_ishole); - good_writes = kmem_zalloc(sizeof (uint64_t), KM_SLEEP); + good_writes = kmem_zalloc(sizeof (uint64_t), KM_PUSHPAGE); vio = zio_null(zio, spa, NULL, (vd->vdev_islog || vd->vdev_aux != NULL) ? vdev_label_sync_ignore_done : vdev_label_sync_top_done, diff --git a/module/zfs/vdev_mirror.c b/module/zfs/vdev_mirror.c index 47181d439..96623d2cf 100644 --- a/module/zfs/vdev_mirror.c +++ b/module/zfs/vdev_mirror.c @@ -79,7 +79,7 @@ vdev_mirror_map_alloc(zio_t *zio) c = BP_GET_NDVAS(zio->io_bp); - mm = kmem_zalloc(offsetof(mirror_map_t, mm_child[c]), KM_SLEEP); + mm = kmem_zalloc(offsetof(mirror_map_t, mm_child[c]), KM_PUSHPAGE); mm->mm_children = c; mm->mm_replacing = B_FALSE; mm->mm_preferred = spa_get_random(c); @@ -106,7 +106,7 @@ vdev_mirror_map_alloc(zio_t *zio) } else { c = vd->vdev_children; - mm = kmem_zalloc(offsetof(mirror_map_t, mm_child[c]), KM_SLEEP); + mm = kmem_zalloc(offsetof(mirror_map_t, mm_child[c]), KM_PUSHPAGE); mm->mm_children = c; mm->mm_replacing = (vd->vdev_ops == &vdev_replacing_ops || vd->vdev_ops == &vdev_spare_ops); diff --git a/module/zfs/vdev_raidz.c b/module/zfs/vdev_raidz.c index b987ac40b..21c6e032b 100644 --- a/module/zfs/vdev_raidz.c +++ b/module/zfs/vdev_raidz.c @@ -456,7 +456,7 @@ vdev_raidz_map_alloc(zio_t *zio, uint64_t unit_shift, uint64_t dcols, ASSERT3U(acols, <=, scols); - rm = kmem_alloc(offsetof(raidz_map_t, rm_col[scols]), KM_SLEEP); + rm = kmem_alloc(offsetof(raidz_map_t, rm_col[scols]), KM_PUSHPAGE); rm->rm_cols = acols; rm->rm_scols = scols; @@ -1196,7 +1196,7 @@ vdev_raidz_matrix_reconstruct(raidz_map_t *rm, int n, int nmissing, size_t psize; psize = sizeof (invlog[0][0]) * n * nmissing; - p = kmem_alloc(psize, KM_SLEEP); + p = kmem_alloc(psize, KM_PUSHPAGE); for (pp = p, i = 0; i < nmissing; i++) { invlog[i] = pp; @@ -1313,7 +1313,7 @@ vdev_raidz_reconstruct_general(raidz_map_t *rm, int *tgts, int ntgts) psize = (sizeof (rows[0][0]) + sizeof (invrows[0][0])) * nmissing_rows * n + sizeof (used[0]) * n; - p = kmem_alloc(psize, KM_SLEEP); + p = kmem_alloc(psize, KM_PUSHPAGE); for (pp = p, i = 0; i < nmissing_rows; i++) { rows[i] = pp; diff --git a/module/zfs/zap.c b/module/zfs/zap.c index fa7e61711..fac54eab0 100644 --- a/module/zfs/zap.c +++ b/module/zfs/zap.c @@ -114,7 +114,7 @@ fzap_upgrade(zap_t *zap, dmu_tx_t *tx, zap_flags_t flags) 1<l_dbuf = db; l->l_phys = db->db_data; @@ -390,7 +390,7 @@ static zap_leaf_t * zap_create_leaf(zap_t *zap, dmu_tx_t *tx) { void *winner; - zap_leaf_t *l = kmem_alloc(sizeof (zap_leaf_t), KM_SLEEP); + zap_leaf_t *l = kmem_alloc(sizeof (zap_leaf_t), KM_PUSHPAGE); ASSERT(RW_WRITE_HELD(&zap->zap_rwlock)); @@ -452,7 +452,7 @@ zap_open_leaf(uint64_t blkid, dmu_buf_t *db) ASSERT(blkid != 0); - l = kmem_alloc(sizeof (zap_leaf_t), KM_SLEEP); + l = kmem_alloc(sizeof (zap_leaf_t), KM_PUSHPAGE); rw_init(&l->l_rwlock, NULL, RW_DEFAULT, NULL); rw_enter(&l->l_rwlock, RW_WRITER); l->l_blkid = blkid; @@ -957,7 +957,7 @@ zap_value_search(objset_t *os, uint64_t zapobj, uint64_t value, uint64_t mask, if (mask == 0) mask = -1ULL; - za = kmem_alloc(sizeof (zap_attribute_t), KM_SLEEP); + za = kmem_alloc(sizeof (zap_attribute_t), KM_PUSHPAGE); for (zap_cursor_init(&zc, os, zapobj); (err = zap_cursor_retrieve(&zc, za)) == 0; zap_cursor_advance(&zc)) { diff --git a/module/zfs/zap_micro.c b/module/zfs/zap_micro.c index 3072475f2..178ab0271 100644 --- a/module/zfs/zap_micro.c +++ b/module/zfs/zap_micro.c @@ -172,7 +172,7 @@ zap_name_free(zap_name_t *zn) zap_name_t * zap_name_alloc(zap_t *zap, const char *key, matchtype_t mt) { - zap_name_t *zn = kmem_alloc(sizeof (zap_name_t), KM_SLEEP); + zap_name_t *zn = kmem_alloc(sizeof (zap_name_t), KM_PUSHPAGE); zn->zn_zap = zap; zn->zn_key_intlen = sizeof (*key); @@ -271,7 +271,7 @@ mze_insert(zap_t *zap, int chunkid, uint64_t hash) ASSERT(zap->zap_ismicro); ASSERT(RW_WRITE_HELD(&zap->zap_rwlock)); - mze = kmem_alloc(sizeof (mzap_ent_t), KM_SLEEP); + mze = kmem_alloc(sizeof (mzap_ent_t), KM_PUSHPAGE); mze->mze_chunkid = chunkid; mze->mze_hash = hash; mze->mze_cd = MZE_PHYS(zap, mze)->mze_cd; @@ -365,7 +365,7 @@ mzap_open(objset_t *os, uint64_t obj, dmu_buf_t *db) ASSERT3U(MZAP_ENT_LEN, ==, sizeof (mzap_ent_phys_t)); - zap = kmem_zalloc(sizeof (zap_t), KM_SLEEP); + zap = kmem_zalloc(sizeof (zap_t), KM_PUSHPAGE); rw_init(&zap->zap_rwlock, NULL, RW_DEFAULT, NULL); rw_enter(&zap->zap_rwlock, RW_WRITER); zap->zap_objset = os; diff --git a/module/zfs/zfs_acl.c b/module/zfs/zfs_acl.c index df690b364..98c001950 100644 --- a/module/zfs/zfs_acl.c +++ b/module/zfs/zfs_acl.c @@ -452,7 +452,7 @@ zfs_acl_alloc(int vers) { zfs_acl_t *aclp; - aclp = kmem_zalloc(sizeof (zfs_acl_t), KM_SLEEP); + aclp = kmem_zalloc(sizeof (zfs_acl_t), KM_PUSHPAGE); list_create(&aclp->z_acl, sizeof (zfs_acl_node_t), offsetof(zfs_acl_node_t, z_next)); aclp->z_version = vers; @@ -468,9 +468,9 @@ zfs_acl_node_alloc(size_t bytes) { zfs_acl_node_t *aclnode; - aclnode = kmem_zalloc(sizeof (zfs_acl_node_t), KM_SLEEP); + aclnode = kmem_zalloc(sizeof (zfs_acl_node_t), KM_PUSHPAGE); if (bytes) { - aclnode->z_acldata = kmem_alloc(bytes, KM_SLEEP); + aclnode->z_acldata = kmem_alloc(bytes, KM_PUSHPAGE); aclnode->z_allocdata = aclnode->z_acldata; aclnode->z_allocsize = bytes; aclnode->z_size = bytes; diff --git a/module/zfs/zfs_fm.c b/module/zfs/zfs_fm.c index 7801837f1..0b98231ec 100644 --- a/module/zfs/zfs_fm.c +++ b/module/zfs/zfs_fm.c @@ -519,7 +519,7 @@ annotate_ecksum(nvlist_t *ereport, zio_bad_cksum_t *info, size_t offset = 0; ssize_t start = -1; - zfs_ecksum_info_t *eip = kmem_zalloc(sizeof (*eip), KM_SLEEP); + zfs_ecksum_info_t *eip = kmem_zalloc(sizeof (*eip), KM_PUSHPAGE); /* don't do any annotation for injected checksum errors */ if (info != NULL && info->zbc_injected) @@ -688,7 +688,7 @@ zfs_ereport_start_checksum(spa_t *spa, vdev_t *vd, struct zio *zio, uint64_t offset, uint64_t length, void *arg, zio_bad_cksum_t *info) { - zio_cksum_report_t *report = kmem_zalloc(sizeof (*report), KM_SLEEP); + zio_cksum_report_t *report = kmem_zalloc(sizeof (*report), KM_PUSHPAGE); if (zio->io_vsd != NULL) zio->io_vsd_ops->vsd_cksum_report(zio, report, arg); @@ -697,7 +697,7 @@ zfs_ereport_start_checksum(spa_t *spa, vdev_t *vd, /* copy the checksum failure information if it was provided */ if (info != NULL) { - report->zcr_ckinfo = kmem_zalloc(sizeof (*info), KM_SLEEP); + report->zcr_ckinfo = kmem_zalloc(sizeof (*info), KM_PUSHPAGE); bcopy(info, report->zcr_ckinfo, sizeof (*info)); } diff --git a/module/zfs/zfs_rlock.c b/module/zfs/zfs_rlock.c index f3ada1706..4f34b3758 100644 --- a/module/zfs/zfs_rlock.c +++ b/module/zfs/zfs_rlock.c @@ -426,7 +426,7 @@ zfs_range_lock(znode_t *zp, uint64_t off, uint64_t len, rl_type_t type) ASSERT(type == RL_READER || type == RL_WRITER || type == RL_APPEND); - new = kmem_alloc(sizeof (rl_t), KM_SLEEP); + new = kmem_alloc(sizeof (rl_t), KM_PUSHPAGE); new->r_zp = zp; new->r_off = off; if (len + off < off) /* overflow */ diff --git a/module/zfs/zfs_znode.c b/module/zfs/zfs_znode.c index 3a6872f3e..33fb0083b 100644 --- a/module/zfs/zfs_znode.c +++ b/module/zfs/zfs_znode.c @@ -627,7 +627,7 @@ zfs_mknode(znode_t *dzp, vattr_t *vap, dmu_tx_t *tx, cred_t *cr, * order for DMU_OT_ZNODE is critical since it needs to be constructed * in the old znode_phys_t format. Don't change this ordering */ - sa_attrs = kmem_alloc(sizeof(sa_bulk_attr_t) * ZPL_END, KM_SLEEP); + sa_attrs = kmem_alloc(sizeof(sa_bulk_attr_t) * ZPL_END, KM_PUSHPAGE); if (obj_type == DMU_OT_ZNODE) { SA_ADD_BULK_ATTR(sa_attrs, cnt, SA_ZPL_ATIME(zsb), @@ -1502,13 +1502,13 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx) vattr.va_uid = crgetuid(cr); vattr.va_gid = crgetgid(cr); - rootzp = kmem_cache_alloc(znode_cache, KM_SLEEP); + rootzp = kmem_cache_alloc(znode_cache, KM_PUSHPAGE); rootzp->z_moved = 0; rootzp->z_unlinked = 0; rootzp->z_atime_dirty = 0; rootzp->z_is_sa = USE_SA(version, os); - zsb = kmem_zalloc(sizeof (zfs_sb_t), KM_SLEEP); + zsb = kmem_zalloc(sizeof (zfs_sb_t), KM_PUSHPAGE); zsb->z_os = os; zsb->z_parent = zsb; zsb->z_version = version; @@ -1516,7 +1516,7 @@ zfs_create_fs(objset_t *os, cred_t *cr, nvlist_t *zplprops, dmu_tx_t *tx) zsb->z_use_sa = USE_SA(version, os); zsb->z_norm = norm; - sb = kmem_zalloc(sizeof (struct super_block), KM_SLEEP); + sb = kmem_zalloc(sizeof (struct super_block), KM_PUSHPAGE); sb->s_fs_info = zsb; ZTOI(rootzp)->i_sb = sb; diff --git a/module/zfs/zil.c b/module/zfs/zil.c index 9ab02d70c..292aea27d 100644 --- a/module/zfs/zil.c +++ b/module/zfs/zil.c @@ -165,7 +165,7 @@ zil_bp_tree_add(zilog_t *zilog, const blkptr_t *bp) if (avl_find(t, dva, &where) != NULL) return (EEXIST); - zn = kmem_alloc(sizeof (zil_bp_node_t), KM_SLEEP); + zn = kmem_alloc(sizeof (zil_bp_node_t), KM_PUSHPAGE); zn->zn_dva = *dva; avl_insert(t, zn, where); @@ -455,7 +455,7 @@ zil_alloc_lwb(zilog_t *zilog, blkptr_t *bp, uint64_t txg) { lwb_t *lwb; - lwb = kmem_cache_alloc(zil_lwb_cache, KM_SLEEP); + lwb = kmem_cache_alloc(zil_lwb_cache, KM_PUSHPAGE); lwb->lwb_zilog = zilog; lwb->lwb_blk = *bp; lwb->lwb_buf = zio_buf_alloc(BP_GET_LSIZE(bp)); @@ -752,7 +752,7 @@ zil_add_block(zilog_t *zilog, const blkptr_t *bp) for (i = 0; i < ndvas; i++) { zvsearch.zv_vdev = DVA_GET_VDEV(&bp->blk_dva[i]); if (avl_find(t, &zvsearch, &where) == NULL) { - zv = kmem_alloc(sizeof (*zv), KM_SLEEP); + zv = kmem_alloc(sizeof (*zv), KM_PUSHPAGE); zv->zv_vdev = zvsearch.zv_vdev; avl_insert(t, zv, where); } @@ -1277,7 +1277,7 @@ zil_itx_assign(zilog_t *zilog, itx_t *itx, dmu_tx_t *tx) } ASSERT(itxg->itxg_sod == 0); itxg->itxg_txg = txg; - itxs = itxg->itxg_itxs = kmem_zalloc(sizeof (itxs_t), KM_SLEEP); + itxs = itxg->itxg_itxs = kmem_zalloc(sizeof (itxs_t), KM_PUSHPAGE); list_create(&itxs->i_sync_list, sizeof (itx_t), offsetof(itx_t, itx_node)); @@ -1297,7 +1297,7 @@ zil_itx_assign(zilog_t *zilog, itx_t *itx, dmu_tx_t *tx) ian = avl_find(t, &foid, &where); if (ian == NULL) { - ian = kmem_alloc(sizeof (itx_async_node_t), KM_SLEEP); + ian = kmem_alloc(sizeof (itx_async_node_t), KM_PUSHPAGE); list_create(&ian->ia_list, sizeof (itx_t), offsetof(itx_t, itx_node)); ian->ia_foid = foid; @@ -1685,7 +1685,7 @@ zil_alloc(objset_t *os, zil_header_t *zh_phys) zilog_t *zilog; int i; - zilog = kmem_zalloc(sizeof (zilog_t), KM_SLEEP); + zilog = kmem_zalloc(sizeof (zilog_t), KM_PUSHPAGE); zilog->zl_header = zh_phys; zilog->zl_os = os; @@ -2007,7 +2007,7 @@ zil_replay(objset_t *os, void *arg, zil_replay_func_t *replay_func[TX_MAX_TYPE]) zr.zr_replay = replay_func; zr.zr_arg = arg; zr.zr_byteswap = BP_SHOULD_BYTESWAP(&zh->zh_log); - zr.zr_lr = vmem_alloc(2 * SPA_MAXBLOCKSIZE, KM_SLEEP); + zr.zr_lr = vmem_alloc(2 * SPA_MAXBLOCKSIZE, KM_PUSHPAGE); /* * Wait for in-progress removes to sync before starting replay. diff --git a/module/zfs/zvol.c b/module/zfs/zvol.c index 60ff64ebe..22f14a006 100644 --- a/module/zfs/zvol.c +++ b/module/zfs/zvol.c @@ -785,7 +785,7 @@ zvol_get_data(void *arg, lr_write_t *lr, char *buf, zio_t *zio) ASSERT(zio != NULL); ASSERT(size != 0); - zgd = (zgd_t *)kmem_zalloc(sizeof (zgd_t), KM_SLEEP); + zgd = (zgd_t *)kmem_zalloc(sizeof (zgd_t), KM_PUSHPAGE); zgd->zgd_zilog = zv->zv_zilog; zgd->zgd_rl = zfs_range_lock(&zv->zv_znode, offset, size, RL_READER);