From cef125313548787f3cdce26413e52bd6127bbc2d Mon Sep 17 00:00:00 2001 From: Tino Reichardt Date: Wed, 28 Sep 2022 10:54:02 +0200 Subject: [PATCH] Add SHA2 SIMD feature tests for Linux These are added: - zfs_neon_available() for arm and aarch64 - zfs_sha256_available() for arm and aarch64 - zfs_sha512_available() for aarch64 - zfs_shani_available() for x86_64 Tested-by: Rich Ercolani Tested-by: Sebastian Gottschall Reviewed-by: Brian Behlendorf Signed-off-by: Tino Reichardt Co-Authored-By: Sebastian Gottschall Closes #13741 --- include/os/linux/Makefile.am | 1 + include/os/linux/kernel/linux/simd.h | 5 +- include/os/linux/kernel/linux/simd_aarch64.h | 76 +++++++++++++++++-- include/os/linux/kernel/linux/simd_arm.h | 80 ++++++++++++++++++++ include/os/linux/kernel/linux/simd_powerpc.h | 5 -- include/os/linux/kernel/linux/simd_x86.h | 15 ++++ 6 files changed, 169 insertions(+), 13 deletions(-) create mode 100644 include/os/linux/kernel/linux/simd_arm.h diff --git a/include/os/linux/Makefile.am b/include/os/linux/Makefile.am index e821e075d..3830d198d 100644 --- a/include/os/linux/Makefile.am +++ b/include/os/linux/Makefile.am @@ -10,6 +10,7 @@ kernel_linux_HEADERS = \ %D%/kernel/linux/percpu_compat.h \ %D%/kernel/linux/simd.h \ %D%/kernel/linux/simd_aarch64.h \ + %D%/kernel/linux/simd_arm.h \ %D%/kernel/linux/simd_powerpc.h \ %D%/kernel/linux/simd_x86.h \ %D%/kernel/linux/utsname_compat.h \ diff --git a/include/os/linux/kernel/linux/simd.h b/include/os/linux/kernel/linux/simd.h index b83c53688..f4376b218 100644 --- a/include/os/linux/kernel/linux/simd.h +++ b/include/os/linux/kernel/linux/simd.h @@ -28,13 +28,16 @@ #if defined(__x86) #include +#elif defined(__arm__) +#include + #elif defined(__aarch64__) #include #elif defined(__powerpc__) #include -#else +#else #define kfpu_allowed() 0 #define kfpu_begin() do {} while (0) #define kfpu_end() do {} while (0) diff --git a/include/os/linux/kernel/linux/simd_aarch64.h b/include/os/linux/kernel/linux/simd_aarch64.h index d56a093d4..16276b08c 100644 --- a/include/os/linux/kernel/linux/simd_aarch64.h +++ b/include/os/linux/kernel/linux/simd_aarch64.h @@ -18,8 +18,11 @@ * * CDDL HEADER END */ + /* * Copyright (C) 2016 Romain Dolbeau . + * Copyright (C) 2022 Tino Reichardt + * Copyright (C) 2022 Sebastian Gottschall */ /* @@ -31,24 +34,83 @@ * kfpu_end() * kfpu_init() * kfpu_fini() + * + * SIMD support: + * + * Following functions should be called to determine whether CPU feature + * is supported. All functions are usable in kernel and user space. + * If a SIMD algorithm is using more than one instruction set + * all relevant feature test functions should be called. + * + * Supported features: + * zfs_neon_available() + * zfs_sha256_available() + * zfs_sha512_available() */ #ifndef _LINUX_SIMD_AARCH64_H #define _LINUX_SIMD_AARCH64_H -#include - -#if defined(__aarch64__) - #include #include +#include +#include +#include + +#if LINUX_VERSION_CODE >= KERNEL_VERSION(3, 17, 0) +#include +#else +#define sys_reg(op0, op1, crn, crm, op2) ( \ + ((op0) << Op0_shift) | \ + ((op1) << Op1_shift) | \ + ((crn) << CRn_shift) | \ + ((crm) << CRm_shift) | \ + ((op2) << Op2_shift)) +#endif + +#define ID_AA64PFR0_EL1 sys_reg(3, 0, 0, 1, 0) +#define ID_AA64ISAR0_EL1 sys_reg(3, 0, 0, 6, 0) #define kfpu_allowed() 1 #define kfpu_begin() kernel_neon_begin() #define kfpu_end() kernel_neon_end() -#define kfpu_init() 0 -#define kfpu_fini() ((void) 0) +#define kfpu_init() (0) +#define kfpu_fini() do {} while (0) -#endif /* __aarch64__ */ +#define get_ftr(id) { \ + unsigned long __val; \ + asm("mrs %0, "#id : "=r" (__val)); \ + __val; \ +} + +/* + * Check if NEON is available + */ +static inline boolean_t +zfs_neon_available(void) +{ + unsigned long ftr = ((get_ftr(ID_AA64PFR0_EL1)) >> 16) & 0xf; + return (ftr == 0 || ftr == 1); +} + +/* + * Check if SHA256 is available + */ +static inline boolean_t +zfs_sha256_available(void) +{ + unsigned long ftr = ((get_ftr(ID_AA64ISAR0_EL1)) >> 12) & 0x3; + return (ftr & 0x1); +} + +/* + * Check if SHA512 is available + */ +static inline boolean_t +zfs_sha512_available(void) +{ + unsigned long ftr = ((get_ftr(ID_AA64ISAR0_EL1)) >> 12) & 0x3; + return (ftr & 0x2); +} #endif /* _LINUX_SIMD_AARCH64_H */ diff --git a/include/os/linux/kernel/linux/simd_arm.h b/include/os/linux/kernel/linux/simd_arm.h new file mode 100644 index 000000000..c432a6d4a --- /dev/null +++ b/include/os/linux/kernel/linux/simd_arm.h @@ -0,0 +1,80 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or https://opensource.org/licenses/CDDL-1.0. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (C) 2022 Tino Reichardt + */ + +/* + * USER API: + * + * Kernel fpu methods: + * kfpu_allowed() + * kfpu_begin() + * kfpu_end() + * kfpu_init() + * kfpu_fini() + * + * SIMD support: + * + * Following functions should be called to determine whether CPU feature + * is supported. All functions are usable in kernel and user space. + * If a SIMD algorithm is using more than one instruction set + * all relevant feature test functions should be called. + * + * Supported features: + * zfs_neon_available() + * zfs_sha256_available() + */ + +#ifndef _LINUX_SIMD_ARM_H +#define _LINUX_SIMD_ARM_H + +#include +#include +#include +#include + +#define kfpu_allowed() 1 +#define kfpu_begin() kernel_neon_begin() +#define kfpu_end() kernel_neon_end() +#define kfpu_init() (0) +#define kfpu_fini() do {} while (0) + +/* + * Check if NEON is available + */ +static inline boolean_t +zfs_neon_available(void) +{ + return (elf_hwcap & HWCAP_NEON); +} + +/* + * Check if SHA256 is available + */ +static inline boolean_t +zfs_sha256_available(void) +{ + return (elf_hwcap2 & HWCAP2_SHA2); +} + +#endif /* _LINUX_SIMD_ARM_H */ diff --git a/include/os/linux/kernel/linux/simd_powerpc.h b/include/os/linux/kernel/linux/simd_powerpc.h index f1de3ad01..7faee70fe 100644 --- a/include/os/linux/kernel/linux/simd_powerpc.h +++ b/include/os/linux/kernel/linux/simd_powerpc.h @@ -50,9 +50,6 @@ #ifndef _LINUX_SIMD_POWERPC_H #define _LINUX_SIMD_POWERPC_H -/* only for __powerpc__ */ -#if defined(__powerpc__) - #include #include #include @@ -134,6 +131,4 @@ zfs_isa207_available(void) return (cpu_has_feature(CPU_FTR_ARCH_207S)); } -#endif /* defined(__powerpc) */ - #endif /* _LINUX_SIMD_POWERPC_H */ diff --git a/include/os/linux/kernel/linux/simd_x86.h b/include/os/linux/kernel/linux/simd_x86.h index 2f6c3165a..1d77f0487 100644 --- a/include/os/linux/kernel/linux/simd_x86.h +++ b/include/os/linux/kernel/linux/simd_x86.h @@ -53,6 +53,8 @@ * zfs_bmi1_available() * zfs_bmi2_available() * + * zfs_shani_available() + * * zfs_avx512f_available() * zfs_avx512cd_available() * zfs_avx512er_available() @@ -586,6 +588,19 @@ zfs_movbe_available(void) #endif } +/* + * Check if SHA_NI instruction set is available + */ +static inline boolean_t +zfs_shani_available(void) +{ +#if defined(X86_FEATURE_SHA_NI) + return (!!boot_cpu_has(X86_FEATURE_SHA_NI)); +#else + return (B_FALSE); +#endif +} + /* * AVX-512 family of instruction sets: *