diff --git a/include/os/freebsd/Makefile.am b/include/os/freebsd/Makefile.am index 5ddb7cd71..3796f20ae 100644 --- a/include/os/freebsd/Makefile.am +++ b/include/os/freebsd/Makefile.am @@ -50,6 +50,7 @@ noinst_HEADERS = \ %D%/spl/sys/sid.h \ %D%/spl/sys/sig.h \ %D%/spl/sys/simd.h \ + %D%/spl/sys/simd_powerpc.h \ %D%/spl/sys/simd_x86.h \ %D%/spl/sys/spl_condvar.h \ %D%/spl/sys/string.h \ diff --git a/include/os/freebsd/spl/sys/simd.h b/include/os/freebsd/spl/sys/simd.h index 53503e838..3106e4853 100644 --- a/include/os/freebsd/spl/sys/simd.h +++ b/include/os/freebsd/spl/sys/simd.h @@ -26,13 +26,16 @@ * $FreeBSD$ */ - #ifndef _FREEBSD_SIMD_H #define _FREEBSD_SIMD_H + #if defined(__amd64__) || defined(__i386__) #include -#else +#elif defined(__powerpc__) +#include + +#else #define kfpu_allowed() 0 #define kfpu_initialize(tsk) do {} while (0) #define kfpu_begin() do {} while (0) @@ -40,4 +43,5 @@ #define kfpu_init() (0) #define kfpu_fini() do {} while (0) #endif + #endif diff --git a/include/os/freebsd/spl/sys/simd_powerpc.h b/include/os/freebsd/spl/sys/simd_powerpc.h new file mode 100644 index 000000000..b90240580 --- /dev/null +++ b/include/os/freebsd/spl/sys/simd_powerpc.h @@ -0,0 +1,90 @@ +/* + * CDDL HEADER START + * + * The contents of this file are subject to the terms of the + * Common Development and Distribution License (the "License"). + * You may not use this file except in compliance with the License. + * + * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE + * or https://opensource.org/licenses/CDDL-1.0. + * See the License for the specific language governing permissions + * and limitations under the License. + * + * When distributing Covered Code, include this CDDL HEADER in each + * file and include the License file at usr/src/OPENSOLARIS.LICENSE. + * If applicable, add the following below this CDDL HEADER, with the + * fields enclosed by brackets "[]" replaced with your own identifying + * information: Portions Copyright [yyyy] [name of copyright owner] + * + * CDDL HEADER END + */ + +/* + * Copyright (C) 2022 Tino Reichardt + */ + +/* + * USER API: + * + * Kernel fpu methods: + * kfpu_allowed() + * kfpu_begin() + * kfpu_end() + * kfpu_init() + * kfpu_fini() + * + * SIMD support: + * + * Following functions should be called to determine whether CPU feature + * is supported. All functions are usable in kernel and user space. + * If a SIMD algorithm is using more than one instruction set + * all relevant feature test functions should be called. + * + * Supported features: + * zfs_altivec_available() + * zfs_vsx_available() + * zfs_isa207_available() + */ + +#ifndef _FREEBSD_SIMD_POWERPC_H +#define _FREEBSD_SIMD_POWERPC_H + +#include +#include + +#include +#include + +#define kfpu_allowed() 1 +#define kfpu_initialize(tsk) do {} while (0) +#define kfpu_begin() do {} while (0) +#define kfpu_end() do {} while (0) +#define kfpu_init() (0) +#define kfpu_fini() do {} while (0) + +/* + * Check if Altivec is available + */ +static inline boolean_t +zfs_altivec_available(void) +{ + return ((cpu_features & PPC_FEATURE_HAS_ALTIVEC) != 0); +} + +/* + * Check if VSX is available + */ +static inline boolean_t +zfs_vsx_available(void) +{ + return ((cpu_features & PPC_FEATURE_HAS_VSX) != 0); +} + +/* + * Check if POWER ISA 2.07 is available (SHA2) + */ +static inline boolean_t +zfs_isa207_available(void) +{ + return ((cpu_features2 & PPC_FEATURE2_ARCH_2_07) != 0); +} diff --git a/include/os/freebsd/spl/sys/simd_x86.h b/include/os/freebsd/spl/sys/simd_x86.h index 480bfd289..7a0ca243f 100644 --- a/include/os/freebsd/spl/sys/simd_x86.h +++ b/include/os/freebsd/spl/sys/simd_x86.h @@ -77,7 +77,7 @@ __simd_state_enabled(const uint64_t state) boolean_t has_osxsave; uint64_t xcr0; - has_osxsave = !!(cpu_feature2 & CPUID2_OSXSAVE); + has_osxsave = (cpu_feature2 & CPUID2_OSXSAVE) != 0; if (!has_osxsave) return (B_FALSE); @@ -99,7 +99,7 @@ __simd_state_enabled(const uint64_t state) static inline boolean_t zfs_sse_available(void) { - return (!!(cpu_feature & CPUID_SSE)); + return ((cpu_feature & CPUID_SSE) != 0); } /* @@ -108,7 +108,7 @@ zfs_sse_available(void) static inline boolean_t zfs_sse2_available(void) { - return (!!(cpu_feature & CPUID_SSE2)); + return ((cpu_feature & CPUID_SSE2) != 0); } /* @@ -117,7 +117,7 @@ zfs_sse2_available(void) static inline boolean_t zfs_sse3_available(void) { - return (!!(cpu_feature2 & CPUID2_SSE3)); + return ((cpu_feature2 & CPUID2_SSE3) != 0); } /* @@ -126,7 +126,7 @@ zfs_sse3_available(void) static inline boolean_t zfs_ssse3_available(void) { - return (!!(cpu_feature2 & CPUID2_SSSE3)); + return ((cpu_feature2 & CPUID2_SSSE3) != 0); } /* @@ -135,7 +135,7 @@ zfs_ssse3_available(void) static inline boolean_t zfs_sse4_1_available(void) { - return (!!(cpu_feature2 & CPUID2_SSE41)); + return ((cpu_feature2 & CPUID2_SSE41) != 0); } /* @@ -144,7 +144,7 @@ zfs_sse4_1_available(void) static inline boolean_t zfs_sse4_2_available(void) { - return (!!(cpu_feature2 & CPUID2_SSE42)); + return ((cpu_feature2 & CPUID2_SSE42) != 0); } /* @@ -155,7 +155,7 @@ zfs_avx_available(void) { boolean_t has_avx; - has_avx = !!(cpu_feature2 & CPUID2_AVX); + has_avx = (cpu_feature2 & CPUID2_AVX) != 0; return (has_avx && __ymm_enabled()); } @@ -168,7 +168,7 @@ zfs_avx2_available(void) { boolean_t has_avx2; - has_avx2 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX2); + has_avx2 = (cpu_stdext_feature & CPUID_STDEXT_AVX2) != 0; return (has_avx2 && __ymm_enabled()); } @@ -196,7 +196,7 @@ zfs_avx512f_available(void) { boolean_t has_avx512; - has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F); + has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512F) != 0; return (has_avx512 && __zmm_enabled()); } @@ -207,8 +207,8 @@ zfs_avx512cd_available(void) { boolean_t has_avx512; - has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && - !!(cpu_stdext_feature & CPUID_STDEXT_AVX512CD); + has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512F) != 0 && + (cpu_stdext_feature & CPUID_STDEXT_AVX512CD) != 0; return (has_avx512 && __zmm_enabled()); } @@ -219,8 +219,8 @@ zfs_avx512er_available(void) { boolean_t has_avx512; - has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && - !!(cpu_stdext_feature & CPUID_STDEXT_AVX512CD); + has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512F) != 0 && + (cpu_stdext_feature & CPUID_STDEXT_AVX512CD) != 0; return (has_avx512 && __zmm_enabled()); } @@ -231,8 +231,8 @@ zfs_avx512pf_available(void) { boolean_t has_avx512; - has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && - !!(cpu_stdext_feature & CPUID_STDEXT_AVX512PF); + has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512F) != 0 && + (cpu_stdext_feature & CPUID_STDEXT_AVX512PF) != 0; return (has_avx512 && __zmm_enabled()); } @@ -243,7 +243,7 @@ zfs_avx512bw_available(void) { boolean_t has_avx512 = B_FALSE; - has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512BW); + has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512BW) != 0; return (has_avx512 && __zmm_enabled()); } @@ -254,8 +254,8 @@ zfs_avx512dq_available(void) { boolean_t has_avx512; - has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && - !!(cpu_stdext_feature & CPUID_STDEXT_AVX512DQ); + has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512F) != 0 && + (cpu_stdext_feature & CPUID_STDEXT_AVX512DQ) != 0; return (has_avx512 && __zmm_enabled()); } @@ -266,8 +266,8 @@ zfs_avx512vl_available(void) { boolean_t has_avx512; - has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && - !!(cpu_stdext_feature & CPUID_STDEXT_AVX512VL); + has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512F) != 0 && + (cpu_stdext_feature & CPUID_STDEXT_AVX512VL) != 0; return (has_avx512 && __zmm_enabled()); } @@ -278,8 +278,8 @@ zfs_avx512ifma_available(void) { boolean_t has_avx512; - has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && - !!(cpu_stdext_feature & CPUID_STDEXT_AVX512IFMA); + has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512F) != 0 && + (cpu_stdext_feature & CPUID_STDEXT_AVX512IFMA) != 0; return (has_avx512 && __zmm_enabled()); } @@ -290,8 +290,8 @@ zfs_avx512vbmi_available(void) { boolean_t has_avx512; - has_avx512 = !!(cpu_stdext_feature & CPUID_STDEXT_AVX512F) && - !!(cpu_stdext_feature & CPUID_STDEXT_BMI1); + has_avx512 = (cpu_stdext_feature & CPUID_STDEXT_AVX512F) != 0 && + (cpu_stdext_feature & CPUID_STDEXT_BMI1) != 0; return (has_avx512 && __zmm_enabled()); } diff --git a/include/os/linux/kernel/linux/simd_powerpc.h b/include/os/linux/kernel/linux/simd_powerpc.h index 764c5dc51..2a2f92bc4 100644 --- a/include/os/linux/kernel/linux/simd_powerpc.h +++ b/include/os/linux/kernel/linux/simd_powerpc.h @@ -21,6 +21,7 @@ /* * Copyright (C) 2019 Romain Dolbeau * + * Copyright (C) 2022 Tino Reichardt */ /* @@ -41,7 +42,9 @@ * all relevant feature test functions should be called. * * Supported features: - * zfs_altivec_available() + * zfs_altivec_available() + * zfs_vsx_available() + * zfs_isa207_available() */ #ifndef _LINUX_SIMD_POWERPC_H @@ -57,73 +60,65 @@ #include #include +#if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 7, 0) +#include +#else +#include +#endif + #define kfpu_allowed() 1 #if LINUX_VERSION_CODE >= KERNEL_VERSION(4, 5, 0) -#define kfpu_end() \ - { \ - disable_kernel_vsx(); \ - disable_kernel_altivec(); \ - preempt_enable(); \ - } #define kfpu_begin() \ { \ preempt_disable(); \ enable_kernel_altivec(); \ enable_kernel_vsx(); \ + enable_kernel_spe(); \ + } +#define kfpu_end() \ + { \ + disable_kernel_spe(); \ + disable_kernel_vsx(); \ + disable_kernel_altivec(); \ + preempt_enable(); \ } #else /* seems that before 4.5 no-one bothered */ #define kfpu_begin() #define kfpu_end() preempt_enable() #endif + #define kfpu_init() 0 #define kfpu_fini() ((void) 0) -static inline boolean_t -zfs_vsx_available(void) -{ - boolean_t res; -#if defined(__powerpc64__) - u64 msr; -#else - u32 msr; -#endif - kfpu_begin(); - __asm volatile("mfmsr %0" : "=r"(msr)); - res = (msr & 0x800000) != 0; - kfpu_end(); - return (res); -} - /* * Check if AltiVec instruction set is available */ static inline boolean_t zfs_altivec_available(void) { - boolean_t res; - /* suggested by macallan at netbsd dot org */ -#if defined(__powerpc64__) - u64 msr; -#else - u32 msr; -#endif - kfpu_begin(); - __asm volatile("mfmsr %0" : "=r"(msr)); - /* - * 64 bits -> need to check bit 38 - * Power ISA Version 3.0B - * p944 - * 32 bits -> Need to check bit 6 - * AltiVec Technology Programming Environments Manual - * p49 (2-9) - * They are the same, as ppc counts 'backward' ... - */ - res = (msr & 0x2000000) != 0; - kfpu_end(); - return (res); + return (cpu_has_feature(CPU_FTR_ALTIVEC)); } + +/* + * Check if VSX is available + */ +static inline boolean_t +zfs_vsx_available(void) +{ + return (cpu_has_feature(CPU_FTR_VSX)); +} + +/* + * Check if POWER ISA 2.07 is available (SHA2) + */ +static inline boolean_t +zfs_isa207_available(void) +{ + return (cpu_has_feature(CPU_FTR_ARCH_207S)); +} + #endif /* defined(__powerpc) */ #endif /* _LINUX_SIMD_POWERPC_H */ diff --git a/lib/libspl/include/sys/simd.h b/lib/libspl/include/sys/simd.h index c9d86a080..c0099dd79 100644 --- a/lib/libspl/include/sys/simd.h +++ b/lib/libspl/include/sys/simd.h @@ -20,8 +20,8 @@ * CDDL HEADER END */ /* - * Copyright 2006 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. + * Copyright (c) 2006 Sun Microsystems, Inc. All rights reserved. + * Copyright (c) 2022 Tino Reichardt */ #ifndef _LIBSPL_SYS_SIMD_H @@ -452,63 +452,60 @@ zfs_avx512vbmi_available(void) #elif defined(__powerpc__) +/* including clashes with AT_UID and others */ +extern unsigned long getauxval(unsigned long type); +#if defined(__FreeBSD__) +#define AT_HWCAP 25 /* CPU feature flags. */ +#define AT_HWCAP2 26 /* CPU feature flags 2. */ +extern int elf_aux_info(int aux, void *buf, int buflen); +static unsigned long getauxval(unsigned long key) +{ + unsigned long val = 0UL; + + if (elf_aux_info((int)key, &val, sizeof (val)) != 0) + return (0UL); + + return (val); +} +#elif defined(__linux__) +#define AT_HWCAP 16 /* CPU feature flags. */ +#define AT_HWCAP2 26 /* CPU feature flags 2. */ +#endif + #define kfpu_allowed() 1 #define kfpu_initialize(tsk) do {} while (0) #define kfpu_begin() do {} while (0) #define kfpu_end() do {} while (0) -/* - * Check if AltiVec instruction set is available - * No easy way beyond 'altivec works' :-( - */ -#include -#include - -#if defined(__ALTIVEC__) && !defined(__FreeBSD__) -static jmp_buf env; -static void sigillhandler(int x) -{ - (void) x; - longjmp(env, 1); -} -#endif - +#define PPC_FEATURE_HAS_ALTIVEC 0x10000000 static inline boolean_t zfs_altivec_available(void) { - boolean_t has_altivec = B_FALSE; -#if defined(__ALTIVEC__) && !defined(__FreeBSD__) - sighandler_t savesig; - savesig = signal(SIGILL, sigillhandler); - if (setjmp(env)) { - signal(SIGILL, savesig); - has_altivec = B_FALSE; - } else { - __asm__ __volatile__("vor 0,0,0\n" : : : "v0"); - signal(SIGILL, savesig); - has_altivec = B_TRUE; - } -#endif - return (has_altivec); + unsigned long hwcap = getauxval(AT_HWCAP); + + return (hwcap & PPC_FEATURE_HAS_ALTIVEC); } + +#define PPC_FEATURE_HAS_VSX 0x00000080 static inline boolean_t zfs_vsx_available(void) { - boolean_t has_vsx = B_FALSE; -#if defined(__ALTIVEC__) && !defined(__FreeBSD__) - sighandler_t savesig; - savesig = signal(SIGILL, sigillhandler); - if (setjmp(env)) { - signal(SIGILL, savesig); - has_vsx = B_FALSE; - } else { - __asm__ __volatile__("xssubsp 0,0,0\n"); - signal(SIGILL, savesig); - has_vsx = B_TRUE; - } -#endif - return (has_vsx); + unsigned long hwcap = getauxval(AT_HWCAP); + + return (hwcap & PPC_FEATURE_HAS_VSX); } + +#define PPC_FEATURE2_ARCH_2_07 0x80000000 +static inline boolean_t +zfs_isa207_available(void) +{ + unsigned long hwcap = getauxval(AT_HWCAP); + unsigned long hwcap2 = getauxval(AT_HWCAP2); + + return ((hwcap & PPC_FEATURE_HAS_VSX) && + (hwcap2 & PPC_FEATURE2_ARCH_2_07)); +} + #else #define kfpu_allowed() 0