diff --git a/include/os/linux/spl/sys/shrinker.h b/include/os/linux/spl/sys/shrinker.h index 4193bc5c4..e608cd30c 100644 --- a/include/os/linux/spl/sys/shrinker.h +++ b/include/os/linux/spl/sys/shrinker.h @@ -32,46 +32,37 @@ * Due to frequent changes in the shrinker API the following * compatibility wrappers should be used. They are as follows: * - * SPL_SHRINKER_DECLARE is used to declare the shrinker which is - * passed to spl_register_shrinker()/spl_unregister_shrinker(). Use - * shrinker_name to set the shrinker variable name, shrinker_callback - * to set the callback function, and seek_cost to define the cost of - * reclaiming an object. - * - * SPL_SHRINKER_DECLARE(shrinker_name, shrinker_callback, seek_cost); - * - * SPL_SHRINKER_CALLBACK_FWD_DECLARE is used when a forward declaration - * of the shrinker callback function is required. Only the callback - * function needs to be passed. - * - * SPL_SHRINKER_CALLBACK_FWD_DECLARE(shrinker_callback); - * - * SPL_SHRINKER_CALLBACK_WRAPPER is used to declare the callback function - * which is registered with the shrinker. This function will call your - * custom shrinker which must use the following prototype. Notice the - * leading __'s, these must be appended to the callback_function name. - * - * int __shrinker_callback(struct shrinker *, struct shrink_control *) - * SPL_SHRINKER_CALLBACK_WRAPPER(shrinker_callback);a + * SPL_SHRINKER_DECLARE(varname, countfunc, scanfunc, seek_cost); * + * SPL_SHRINKER_DECLARE is used to declare a shrinker with the name varname, + * which is passed to spl_register_shrinker()/spl_unregister_shrinker(). + * The countfunc returns the number of free-able objects. + * The scanfunc returns the number of objects that were freed. + * The callbacks can return SHRINK_STOP if further calls can't make any more + * progress. Note that a return value of SHRINK_EMPTY is currently not + * supported. * * Example: * - * SPL_SHRINKER_CALLBACK_FWD_DECLARE(my_shrinker_fn); - * SPL_SHRINKER_DECLARE(my_shrinker, my_shrinker_fn, 1); - * - * static int - * __my_shrinker_fn(struct shrinker *shrink, struct shrink_control *sc) + * static unsigned long + * my_count(struct shrinker *shrink, struct shrink_control *sc) * { - * if (sc->nr_to_scan) { - * ...scan objects in the cache and reclaim them... - * } - * * ...calculate number of objects in the cache... * * return (number of objects in the cache); * } - * SPL_SHRINKER_CALLBACK_WRAPPER(my_shrinker_fn); + * + * static unsigned long + * my_scan(struct shrinker *shrink, struct shrink_control *sc) + * { + * ...scan objects in the cache and reclaim them... + * } + * + * SPL_SHRINKER_DECLARE(my_shrinker, my_count, my_scan, DEFAULT_SEEKS); + * + * void my_init_func(void) { + * spl_register_shrinker(&my_shrinker); + * } */ #define spl_register_shrinker(x) register_shrinker(x) @@ -81,60 +72,34 @@ * Linux 3.0 to 3.11 Shrinker API Compatibility. */ #if defined(HAVE_SINGLE_SHRINKER_CALLBACK) -#define SPL_SHRINKER_DECLARE(s, x, y) \ -static struct shrinker s = { \ - .shrink = x, \ - .seeks = y \ -} - -#define SPL_SHRINKER_CALLBACK_FWD_DECLARE(fn) \ -static int fn(struct shrinker *, struct shrink_control *) - -#define SPL_SHRINKER_CALLBACK_WRAPPER(fn) \ +#define SPL_SHRINKER_DECLARE(varname, countfunc, scanfunc, seek_cost) \ static int \ -fn(struct shrinker *shrink, struct shrink_control *sc) \ +__ ## varname ## _wrapper(struct shrinker *shrink, struct shrink_control *sc)\ { \ - return (__ ## fn(shrink, sc)); \ + if (sc->nr_to_scan != 0) { \ + (void) scanfunc(shrink, sc); \ + } \ + return (countfunc(shrink, sc)); \ +} \ + \ +static struct shrinker varname = { \ + .shrink = __ ## varname ## _wrapper, \ + .seeks = seek_cost \ } +#define SHRINK_STOP (-1) + /* * Linux 3.12 and later Shrinker API Compatibility. */ #elif defined(HAVE_SPLIT_SHRINKER_CALLBACK) -#define SPL_SHRINKER_DECLARE(s, x, y) \ -static struct shrinker s = { \ - .count_objects = x ## _count_objects, \ - .scan_objects = x ## _scan_objects, \ - .seeks = y \ +#define SPL_SHRINKER_DECLARE(varname, countfunc, scanfunc, seek_cost) \ +static struct shrinker varname = { \ + .count_objects = countfunc, \ + .scan_objects = scanfunc, \ + .seeks = seek_cost \ } -#define SPL_SHRINKER_CALLBACK_FWD_DECLARE(fn) \ -static unsigned long fn ## _count_objects(struct shrinker *, \ - struct shrink_control *); \ -static unsigned long fn ## _scan_objects(struct shrinker *, \ - struct shrink_control *) - -#define SPL_SHRINKER_CALLBACK_WRAPPER(fn) \ -static unsigned long \ -fn ## _count_objects(struct shrinker *shrink, struct shrink_control *sc)\ -{ \ - int __ret__; \ - \ - sc->nr_to_scan = 0; \ - __ret__ = __ ## fn(NULL, sc); \ - \ - /* Errors may not be returned and must be converted to zeros */ \ - return ((__ret__ < 0) ? 0 : __ret__); \ -} \ - \ -static unsigned long \ -fn ## _scan_objects(struct shrinker *shrink, struct shrink_control *sc) \ -{ \ - int __ret__; \ - \ - __ret__ = __ ## fn(NULL, sc); \ - return ((__ret__ < 0) ? SHRINK_STOP : __ret__); \ -} #else /* * Linux 2.x to 2.6.22, or a newer shrinker API has been introduced. @@ -142,11 +107,4 @@ fn ## _scan_objects(struct shrinker *shrink, struct shrink_control *sc) \ #error "Unknown shrinker callback" #endif -#if defined(HAVE_SPLIT_SHRINKER_CALLBACK) -typedef unsigned long spl_shrinker_t; -#else -typedef int spl_shrinker_t; -#define SHRINK_STOP (-1) -#endif - #endif /* SPL_SHRINKER_H */ diff --git a/module/os/linux/spl/spl-kmem-cache.c b/module/os/linux/spl/spl-kmem-cache.c index 3fab184c8..9506eda36 100644 --- a/module/os/linux/spl/spl-kmem-cache.c +++ b/module/os/linux/spl/spl-kmem-cache.c @@ -190,10 +190,6 @@ taskq_t *spl_kmem_cache_taskq; /* Task queue for aging / reclaim */ static void spl_cache_shrink(spl_kmem_cache_t *skc, void *obj); -SPL_SHRINKER_CALLBACK_FWD_DECLARE(spl_kmem_cache_generic_shrinker); -SPL_SHRINKER_DECLARE(spl_kmem_cache_shrinker, - spl_kmem_cache_generic_shrinker, KMC_DEFAULT_SEEKS); - static void * kv_alloc(spl_kmem_cache_t *skc, int size, int flags) { @@ -1619,23 +1615,27 @@ EXPORT_SYMBOL(spl_kmem_cache_free); * We always attempt to shrink all caches when this generic shrinker * is called. * - * If sc->nr_to_scan is zero, the caller is requesting a query of the - * number of objects which can potentially be freed. If it is nonzero, - * the request is to free that many objects. - * - * Linux kernels >= 3.12 have the count_objects and scan_objects callbacks - * in struct shrinker and also require the shrinker to return the number - * of objects freed. - * - * Older kernels require the shrinker to return the number of freeable - * objects following the freeing of nr_to_free. - * - * Linux semantics differ from those under Solaris, which are to - * free all available objects which may (and probably will) be more - * objects than the requested nr_to_scan. + * The _count() function returns the number of free-able objects. + * The _scan() function returns the number of objects that were freed. */ -static spl_shrinker_t -__spl_kmem_cache_generic_shrinker(struct shrinker *shrink, +static unsigned long +spl_kmem_cache_shrinker_count(struct shrinker *shrink, + struct shrink_control *sc) +{ + spl_kmem_cache_t *skc = NULL; + int alloc = 0; + + down_read(&spl_kmem_cache_sem); + list_for_each_entry(skc, &spl_kmem_cache_list, skc_list) { + alloc += skc->skc_obj_alloc; + } + up_read(&spl_kmem_cache_sem); + + return (MAX(alloc, 0)); +} + +static unsigned long +spl_kmem_cache_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) { spl_kmem_cache_t *skc = NULL; @@ -1644,27 +1644,16 @@ __spl_kmem_cache_generic_shrinker(struct shrinker *shrink, /* * No shrinking in a transaction context. Can cause deadlocks. */ - if (sc->nr_to_scan && spl_fstrans_check()) + if (spl_fstrans_check()) return (SHRINK_STOP); down_read(&spl_kmem_cache_sem); list_for_each_entry(skc, &spl_kmem_cache_list, skc_list) { - if (sc->nr_to_scan) { -#ifdef HAVE_SPLIT_SHRINKER_CALLBACK - uint64_t oldalloc = skc->skc_obj_alloc; - spl_kmem_cache_reap_now(skc, - MAX(sc->nr_to_scan>>fls64(skc->skc_slab_objs), 1)); - if (oldalloc > skc->skc_obj_alloc) - alloc += oldalloc - skc->skc_obj_alloc; -#else - spl_kmem_cache_reap_now(skc, - MAX(sc->nr_to_scan>>fls64(skc->skc_slab_objs), 1)); - alloc += skc->skc_obj_alloc; -#endif /* HAVE_SPLIT_SHRINKER_CALLBACK */ - } else { - /* Request to query number of freeable objects */ - alloc += skc->skc_obj_alloc; - } + uint64_t oldalloc = skc->skc_obj_alloc; + spl_kmem_cache_reap_now(skc, + MAX(sc->nr_to_scan>>fls64(skc->skc_slab_objs), 1)); + if (oldalloc > skc->skc_obj_alloc) + alloc += oldalloc - skc->skc_obj_alloc; } up_read(&spl_kmem_cache_sem); @@ -1674,13 +1663,15 @@ __spl_kmem_cache_generic_shrinker(struct shrinker *shrink, * shrink_slabs() is repeatedly invoked by many cores causing the * system to thrash. */ - if ((spl_kmem_cache_reclaim & KMC_RECLAIM_ONCE) && sc->nr_to_scan) + if (spl_kmem_cache_reclaim & KMC_RECLAIM_ONCE) return (SHRINK_STOP); return (MAX(alloc, 0)); } -SPL_SHRINKER_CALLBACK_WRAPPER(spl_kmem_cache_generic_shrinker); +SPL_SHRINKER_DECLARE(spl_kmem_cache_shrinker, + spl_kmem_cache_shrinker_count, spl_kmem_cache_shrinker_scan, + KMC_DEFAULT_SEEKS); /* * Call the registered reclaim function for a cache. Depending on how @@ -1789,7 +1780,7 @@ spl_kmem_reap(void) sc.nr_to_scan = KMC_REAP_CHUNK; sc.gfp_mask = GFP_KERNEL; - (void) __spl_kmem_cache_generic_shrinker(NULL, &sc); + (void) spl_kmem_cache_shrinker_scan(NULL, &sc); } EXPORT_SYMBOL(spl_kmem_reap); diff --git a/module/os/linux/zfs/arc_os.c b/module/os/linux/zfs/arc_os.c index 9ac4e3221..0c0289db6 100644 --- a/module/os/linux/zfs/arc_os.c +++ b/module/os/linux/zfs/arc_os.c @@ -225,19 +225,17 @@ arc_evictable_memory(void) } /* - * If sc->nr_to_scan is zero, the caller is requesting a query of the - * number of objects which can potentially be freed. If it is nonzero, - * the request is to free that many objects. - * - * Linux kernels >= 3.12 have the count_objects and scan_objects callbacks - * in struct shrinker and also require the shrinker to return the number - * of objects freed. - * - * Older kernels require the shrinker to return the number of freeable - * objects following the freeing of nr_to_free. + * The _count() function returns the number of free-able objects. + * The _scan() function returns the number of objects that were freed. */ -static spl_shrinker_t -__arc_shrinker_func(struct shrinker *shrink, struct shrink_control *sc) +static unsigned long +arc_shrinker_count(struct shrinker *shrink, struct shrink_control *sc) +{ + return (btop((int64_t)arc_evictable_memory())); +} + +static unsigned long +arc_shrinker_scan(struct shrinker *shrink, struct shrink_control *sc) { int64_t pages; @@ -247,8 +245,6 @@ __arc_shrinker_func(struct shrinker *shrink, struct shrink_control *sc) /* Return the potential number of reclaimable pages */ pages = btop((int64_t)arc_evictable_memory()); - if (sc->nr_to_scan == 0) - return (pages); /* Not allowed to perform filesystem reclaim */ if (!(sc->gfp_mask & __GFP_FS)) @@ -288,12 +284,8 @@ __arc_shrinker_func(struct shrinker *shrink, struct shrink_control *sc) if (current_is_kswapd()) arc_kmem_reap_soon(); -#ifdef HAVE_SPLIT_SHRINKER_CALLBACK pages = MAX((int64_t)pages - (int64_t)btop(arc_evictable_memory()), 0); -#else - pages = btop(arc_evictable_memory()); -#endif /* * We've shrunk what we can, wake up threads. */ @@ -318,9 +310,9 @@ __arc_shrinker_func(struct shrinker *shrink, struct shrink_control *sc) return (pages); } -SPL_SHRINKER_CALLBACK_WRAPPER(arc_shrinker_func); -SPL_SHRINKER_DECLARE(arc_shrinker, arc_shrinker_func, DEFAULT_SEEKS); +SPL_SHRINKER_DECLARE(arc_shrinker, + arc_shrinker_count, arc_shrinker_scan, DEFAULT_SEEKS); int arc_memory_throttle(spa_t *spa, uint64_t reserve, uint64_t txg)