c/mirror_zfs
1
0
Fork 0
mirror of https://git.proxmox.com/git/mirror_zfs.git synced 2024-12-26 11:19:32 +03:00
Code Issues Packages Projects Releases Wiki Activity

Improve spl slab cache alloc

Browse Source 
The policy is to try to allocate with KM_NOSLEEP, which will lead to
memory allocation with GFP_ATOMIC, and if it fails, it will launch
an taskq to expand slab space.

This way it should be able to get better NUMA memory locality and
reduce the overhead of context switch.

Signed-off-by: Jinshan Xiong <jinshan.xiong@gmail.com>
Signed-off-by: Brian Behlendorf <behlendorf1@llnl.gov>
Closes #551
This commit is contained in:
Jinshan Xiong 2016-05-19 10:59:40 -07:00 committed by Brian Behlendorf
parent ea5f1a200b
commit 16fc1ec3ba
1 changed files with 35 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

43
module/spl/spl-kmem-cache.c
View File

@ -1149,15 +1149,13 @@ spl_cache_obj(spl_kmem_cache_t *skc, spl_kmem_slab_t *sks)
* It is responsible for allocating a new slab, linking it in to the list
* of partial slabs, and then waking any waiters.
*/
static void
spl_cache_grow_work(void *data)
static int
__spl_cache_grow(spl_kmem_cache_t *skc, int flags)
{
spl_kmem_alloc_t *ska = (spl_kmem_alloc_t *)data;
spl_kmem_cache_t *skc = ska->ska_cache;
spl_kmem_slab_t *sks;
fstrans_cookie_t cookie = spl_fstrans_mark();
sks = spl_slab_alloc(skc, ska->ska_flags);
sks = spl_slab_alloc(skc, flags);
spl_fstrans_unmark(cookie);
spin_lock(&skc->skc_lock);
@ -1165,15 +1163,29 @@ spl_cache_grow_work(void *data)
skc->skc_slab_total++;
skc->skc_obj_total += sks->sks_objs;
list_add_tail(&sks->sks_list, &skc->skc_partial_list);
smp_mb__before_atomic();
clear_bit(KMC_BIT_DEADLOCKED, &skc->skc_flags);
smp_mb__after_atomic();
wake_up_all(&skc->skc_waitq);
}
spin_unlock(&skc->skc_lock);
return (sks == NULL ? -ENOMEM : 0);
}
static void
spl_cache_grow_work(void *data)
{
spl_kmem_alloc_t *ska = (spl_kmem_alloc_t *)data;
spl_kmem_cache_t *skc = ska->ska_cache;
(void)__spl_cache_grow(skc, ska->ska_flags);
atomic_dec(&skc->skc_ref);
smp_mb__before_atomic();
clear_bit(KMC_BIT_GROWING, &skc->skc_flags);
clear_bit(KMC_BIT_DEADLOCKED, &skc->skc_flags);
smp_mb__after_atomic();
wake_up_all(&skc->skc_waitq);
spin_unlock(&skc->skc_lock);
kfree(ska);
}
@ -1213,6 +1225,21 @@ spl_cache_grow(spl_kmem_cache_t *skc, int flags, void **obj)
return (rc ? rc : -EAGAIN);
}
/*
* To reduce the overhead of context switch and improve NUMA locality,
* it tries to allocate a new slab in the current process context with
* KM_NOSLEEP flag. If it fails, it will launch a new taskq to do the
* allocation.
*
* However, this can't be applied to KVM_VMEM due to a bug that
* __vmalloc() doesn't honor gfp flags in page table allocation.
*/
if (!(skc->skc_flags & KMC_VMEM)) {
rc = __spl_cache_grow(skc, flags | KM_NOSLEEP);
if (rc == 0)
return (0);
}
/*
* This is handled by dispatching a work request to the global work
* queue. This allows us to asynchronously allocate a new slab while