From 35155c013240ce14860b43ebc4803c2a5eea78f8 Mon Sep 17 00:00:00 2001 From: Fabian-Gruenbichler Date: Tue, 10 Dec 2019 21:53:25 +0100 Subject: [PATCH] SIMD: Use alloc_pages_node to force alignment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit fxsave and xsave require the target address to be 16-/64-byte aligned. kmalloc(_node) does not (yet) offer such fine-grained control over alignment[0,1], even though it does "the right thing" most of the time for power-of-2 sizes. unfortunately, alignment is completely off when using certain debugging or hardening features/configs, such as KASAN, slub_debug=Z or the not-yet-upstream SLAB_CANARY. Use alloc_pages_node() instead which allows us to allocate page-aligned memory. Since fpregs_state is padded to a full page anyway, and this code is only relevant for x86 which has 4k pages, this approach should not allocate any unnecessary memory but still guarantee the needed alignment. 0: https://lwn.net/Articles/787740/ 1: https://lore.kernel.org/linux-block/20190826111627.7505-1-vbabka@suse.cz/ Reviewed-by: Tony Hutter Signed-off-by: Fabian Grünbichler Signed-off-by: Brian Behlendorf Closes #9608 Closes #9674 --- include/linux/simd_x86.h | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/include/linux/simd_x86.h b/include/linux/simd_x86.h index 69dbd5579..1bde1d7c9 100644 --- a/include/linux/simd_x86.h +++ b/include/linux/simd_x86.h @@ -144,6 +144,8 @@ */ #if defined(HAVE_KERNEL_FPU_INTERNAL) +#include + extern union fpregs_state **zfs_kfpu_fpregs; /* @@ -156,7 +158,8 @@ kfpu_fini(void) for_each_possible_cpu(cpu) { if (zfs_kfpu_fpregs[cpu] != NULL) { - kfree(zfs_kfpu_fpregs[cpu]); + free_pages((unsigned long)zfs_kfpu_fpregs[cpu], + get_order(sizeof (union fpregs_state))); } } @@ -166,20 +169,28 @@ kfpu_fini(void) static inline int kfpu_init(void) { - int cpu; - zfs_kfpu_fpregs = kzalloc(num_possible_cpus() * sizeof (union fpregs_state *), GFP_KERNEL); if (zfs_kfpu_fpregs == NULL) return (-ENOMEM); + /* + * The fxsave and xsave operations require 16-/64-byte alignment of + * the target memory. Since kmalloc() provides no alignment + * guarantee instead use alloc_pages_node(). + */ + unsigned int order = get_order(sizeof (union fpregs_state)); + int cpu; + for_each_possible_cpu(cpu) { - zfs_kfpu_fpregs[cpu] = kmalloc_node(sizeof (union fpregs_state), - GFP_KERNEL | __GFP_ZERO, cpu_to_node(cpu)); - if (zfs_kfpu_fpregs[cpu] == NULL) { + struct page *page = alloc_pages_node(cpu_to_node(cpu), + GFP_KERNEL | __GFP_ZERO, order); + if (page == NULL) { kfpu_fini(); return (-ENOMEM); } + + zfs_kfpu_fpregs[cpu] = page_address(page); } return (0);