diff --git a/include/os/linux/kernel/linux/simd_x86.h b/include/os/linux/kernel/linux/simd_x86.h index e8f1ce558..42a0affb2 100644 --- a/include/os/linux/kernel/linux/simd_x86.h +++ b/include/os/linux/kernel/linux/simd_x86.h @@ -315,19 +315,19 @@ kfpu_begin(void) * FPU state to be correctly preserved and restored. */ uint8_t *state = zfs_kfpu_fpregs[smp_processor_id()]; -#if defined(HAVE_XSAVES) +#if HAVE_SIMD(XSAVES) if (static_cpu_has(X86_FEATURE_XSAVES)) { kfpu_do_xsave("xsaves", state, ~XFEATURE_MASK_XTILE); return; } #endif -#if defined(HAVE_XSAVEOPT) +#if HAVE_SIMD(XSAVEOPT) if (static_cpu_has(X86_FEATURE_XSAVEOPT)) { kfpu_do_xsave("xsaveopt", state, ~XFEATURE_MASK_XTILE); return; } #endif -#if defined(HAVE_XSAVE) +#if HAVE_SIMD(XSAVE) if (static_cpu_has(X86_FEATURE_XSAVE)) { kfpu_do_xsave("xsave", state, ~XFEATURE_MASK_XTILE); return; @@ -380,13 +380,13 @@ static inline void kfpu_end(void) { uint8_t *state = zfs_kfpu_fpregs[smp_processor_id()]; -#if defined(HAVE_XSAVES) +#if HAVE_SIMD(XSAVES) if (static_cpu_has(X86_FEATURE_XSAVES)) { kfpu_do_xrstor("xrstors", state, ~XFEATURE_MASK_XTILE); goto out; } #endif -#if defined(HAVE_XSAVE) +#if HAVE_SIMD(XSAVE) if (static_cpu_has(X86_FEATURE_XSAVE)) { kfpu_do_xrstor("xrstor", state, ~XFEATURE_MASK_XTILE); goto out; diff --git a/include/sys/vdev_raidz_impl.h b/include/sys/vdev_raidz_impl.h index 8c8dcfb07..55885dbd7 100644 --- a/include/sys/vdev_raidz_impl.h +++ b/include/sys/vdev_raidz_impl.h @@ -172,19 +172,19 @@ typedef struct reflow_node { extern const raidz_impl_ops_t vdev_raidz_scalar_impl; extern boolean_t raidz_will_scalar_work(void); -#if defined(__x86_64) && defined(HAVE_SSE2) /* only x86_64 for now */ +#if defined(__x86_64) && HAVE_SIMD(SSE2) /* only x86_64 for now */ extern const raidz_impl_ops_t vdev_raidz_sse2_impl; #endif -#if defined(__x86_64) && defined(HAVE_SSSE3) /* only x86_64 for now */ +#if defined(__x86_64) && HAVE_SIMD(SSSE3) /* only x86_64 for now */ extern const raidz_impl_ops_t vdev_raidz_ssse3_impl; #endif -#if defined(__x86_64) && defined(HAVE_AVX2) /* only x86_64 for now */ +#if defined(__x86_64) && HAVE_SIMD(AVX2) /* only x86_64 for now */ extern const raidz_impl_ops_t vdev_raidz_avx2_impl; #endif -#if defined(__x86_64) && defined(HAVE_AVX512F) /* only x86_64 for now */ +#if defined(__x86_64) && HAVE_SIMD(AVX512F) /* only x86_64 for now */ extern const raidz_impl_ops_t vdev_raidz_avx512f_impl; #endif -#if defined(__x86_64) && defined(HAVE_AVX512BW) /* only x86_64 for now */ +#if defined(__x86_64) && HAVE_SIMD(AVX512BW) /* only x86_64 for now */ extern const raidz_impl_ops_t vdev_raidz_avx512bw_impl; #endif #if defined(__aarch64__) diff --git a/include/zfs_fletcher.h b/include/zfs_fletcher.h index f3a620a7c..d450f7f5e 100644 --- a/include/zfs_fletcher.h +++ b/include/zfs_fletcher.h @@ -97,13 +97,13 @@ typedef union fletcher_4_ctx { zio_cksum_t scalar; zfs_fletcher_superscalar_t superscalar[4]; -#if defined(HAVE_SSE2) || (defined(HAVE_SSE2) && defined(HAVE_SSSE3)) +#if HAVE_SIMD(SSE2) || (HAVE_SIMD(SSE2) && HAVE_SIMD(SSSE3)) zfs_fletcher_sse_t sse[4]; #endif -#if defined(HAVE_AVX) && defined(HAVE_AVX2) +#if HAVE_SIMD(AVX) && HAVE_SIMD(AVX2) zfs_fletcher_avx_t avx[4]; #endif -#if defined(__x86_64) && defined(HAVE_AVX512F) +#if defined(__x86_64) && HAVE_SIMD(AVX512F) zfs_fletcher_avx512_t avx512[4]; #endif #if defined(__aarch64__) @@ -134,23 +134,23 @@ typedef struct fletcher_4_func { _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_superscalar_ops; _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_superscalar4_ops; -#if defined(HAVE_SSE2) +#if HAVE_SIMD(SSE2) _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_sse2_ops; #endif -#if defined(HAVE_SSE2) && defined(HAVE_SSSE3) +#if HAVE_SIMD(SSE2) && HAVE_SIMD(SSSE3) _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_ssse3_ops; #endif -#if defined(HAVE_AVX) && defined(HAVE_AVX2) +#if HAVE_SIMD(AVX) && HAVE_SIMD(AVX2) _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx2_ops; #endif -#if defined(__x86_64) && defined(HAVE_AVX512F) +#if defined(__x86_64) && HAVE_SIMD(AVX512F) _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx512f_ops; #endif -#if defined(__x86_64) && defined(HAVE_AVX512BW) +#if defined(__x86_64) && HAVE_SIMD(AVX512BW) _ZFS_FLETCHER_H const fletcher_4_ops_t fletcher_4_avx512bw_ops; #endif diff --git a/module/icp/algs/aes/aes_impl.c b/module/icp/algs/aes/aes_impl.c index b652c8c72..fff672bc4 100644 --- a/module/icp/algs/aes/aes_impl.c +++ b/module/icp/algs/aes/aes_impl.c @@ -231,7 +231,7 @@ static const aes_impl_ops_t *aes_all_impl[] = { #if defined(__x86_64) &aes_x86_64_impl, #endif -#if defined(__x86_64) && defined(HAVE_AES) +#if defined(__x86_64) && HAVE_SIMD(AES) &aes_aesni_impl, #endif }; @@ -315,7 +315,7 @@ aes_impl_init(void) * hardware accelerated version is the fastest. */ #if defined(__x86_64) -#if defined(HAVE_AES) +#if HAVE_SIMD(AES) if (aes_aesni_impl.is_supported()) { memcpy(&aes_fastest_impl, &aes_aesni_impl, sizeof (aes_fastest_impl)); diff --git a/module/icp/algs/aes/aes_impl_aesni.c b/module/icp/algs/aes/aes_impl_aesni.c index 1dbcb3b71..2cffaebb2 100644 --- a/module/icp/algs/aes/aes_impl_aesni.c +++ b/module/icp/algs/aes/aes_impl_aesni.c @@ -23,7 +23,7 @@ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. */ -#if defined(__x86_64) && defined(HAVE_AES) +#if defined(__x86_64) && HAVE_SIMD(AES) #include #include @@ -123,4 +123,4 @@ const aes_impl_ops_t aes_aesni_impl = { .name = "aesni" }; -#endif /* defined(__x86_64) && defined(HAVE_AES) */ +#endif /* defined(__x86_64) && HAVE_SIMD(AES) */ diff --git a/module/icp/algs/blake3/blake3_impl.c b/module/icp/algs/blake3/blake3_impl.c index 50c2df0d5..735bed2fc 100644 --- a/module/icp/algs/blake3/blake3_impl.c +++ b/module/icp/algs/blake3/blake3_impl.c @@ -32,7 +32,7 @@ #include "blake3_impl.h" #if defined(__aarch64__) || \ - (defined(__x86_64) && defined(HAVE_SSE2)) || \ + (defined(__x86_64) && HAVE_SIMD(SSE2)) || \ (defined(__PPC64__) && defined(__LITTLE_ENDIAN__)) extern void ASMABI zfs_blake3_compress_in_place_sse2(uint32_t cv[8], @@ -98,7 +98,7 @@ const blake3_ops_t blake3_sse2_impl = { #endif #if defined(__aarch64__) || \ - (defined(__x86_64) && defined(HAVE_SSE2)) || \ + (defined(__x86_64) && HAVE_SIMD(SSE2)) || \ (defined(__PPC64__) && defined(__LITTLE_ENDIAN__)) extern void ASMABI zfs_blake3_compress_in_place_sse41(uint32_t cv[8], @@ -163,7 +163,7 @@ const blake3_ops_t blake3_sse41_impl = { }; #endif -#if defined(__x86_64) && defined(HAVE_SSE4_1) && defined(HAVE_AVX2) +#if defined(__x86_64) && HAVE_SIMD(SSE4_1) && HAVE_SIMD(AVX2) extern void ASMABI zfs_blake3_hash_many_avx2(const uint8_t * const *inputs, size_t num_inputs, size_t blocks, const uint32_t key[8], uint64_t counter, boolean_t increment_counter, uint8_t flags, @@ -196,7 +196,7 @@ blake3_avx2_impl = { }; #endif -#if defined(__x86_64) && defined(HAVE_AVX512F) && defined(HAVE_AVX512VL) +#if defined(__x86_64) && HAVE_SIMD(AVX512F) && HAVE_SIMD(AVX512VL) extern void ASMABI zfs_blake3_compress_in_place_avx512(uint32_t cv[8], const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len, uint64_t counter, uint8_t flags); @@ -259,19 +259,19 @@ extern const blake3_ops_t blake3_generic_impl; static const blake3_ops_t *const blake3_impls[] = { &blake3_generic_impl, #if defined(__aarch64__) || \ - (defined(__x86_64) && defined(HAVE_SSE2)) || \ + (defined(__x86_64) && HAVE_SIMD(SSE2)) || \ (defined(__PPC64__) && defined(__LITTLE_ENDIAN__)) &blake3_sse2_impl, #endif #if defined(__aarch64__) || \ - (defined(__x86_64) && defined(HAVE_SSE4_1)) || \ + (defined(__x86_64) && HAVE_SIMD(SSE4_1)) || \ (defined(__PPC64__) && defined(__LITTLE_ENDIAN__)) &blake3_sse41_impl, #endif -#if defined(__x86_64) && defined(HAVE_SSE4_1) && defined(HAVE_AVX2) +#if defined(__x86_64) && HAVE_SIMD(SSE4_1) && HAVE_SIMD(AVX2) &blake3_avx2_impl, #endif -#if defined(__x86_64) && defined(HAVE_AVX512F) && defined(HAVE_AVX512VL) +#if defined(__x86_64) && HAVE_SIMD(AVX512F) && HAVE_SIMD(AVX512VL) &blake3_avx512_impl, #endif }; diff --git a/module/icp/algs/modes/gcm.c b/module/icp/algs/modes/gcm.c index ab2c5a49f..7d92b208f 100644 --- a/module/icp/algs/modes/gcm.c +++ b/module/icp/algs/modes/gcm.c @@ -714,7 +714,7 @@ static gcm_impl_ops_t gcm_fastest_impl = { /* All compiled in implementations */ static const gcm_impl_ops_t *gcm_all_impl[] = { &gcm_generic_impl, -#if defined(__x86_64) && defined(HAVE_PCLMULQDQ) +#if defined(__x86_64) && HAVE_SIMD(PCLMULQDQ) &gcm_pclmulqdq_impl, #endif }; @@ -801,7 +801,7 @@ gcm_impl_init(void) * Set the fastest implementation given the assumption that the * hardware accelerated version is the fastest. */ -#if defined(__x86_64) && defined(HAVE_PCLMULQDQ) +#if defined(__x86_64) && HAVE_SIMD(PCLMULQDQ) if (gcm_pclmulqdq_impl.is_supported()) { memcpy(&gcm_fastest_impl, &gcm_pclmulqdq_impl, sizeof (gcm_fastest_impl)); @@ -827,7 +827,7 @@ gcm_impl_init(void) } else #endif if (gcm_avx_will_work()) { -#ifdef HAVE_MOVBE +#if HAVE_SIMD(MOVBE) if (zfs_movbe_available() == B_TRUE) { atomic_swap_32(&gcm_avx_can_use_movbe, B_TRUE); } diff --git a/module/icp/algs/modes/gcm_pclmulqdq.c b/module/icp/algs/modes/gcm_pclmulqdq.c index 30653589f..53361921d 100644 --- a/module/icp/algs/modes/gcm_pclmulqdq.c +++ b/module/icp/algs/modes/gcm_pclmulqdq.c @@ -23,7 +23,7 @@ * Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved. */ -#if defined(__x86_64) && defined(HAVE_PCLMULQDQ) +#if defined(__x86_64) && HAVE_SIMD(PCLMULQDQ) #include #include @@ -63,4 +63,4 @@ const gcm_impl_ops_t gcm_pclmulqdq_impl = { .name = "pclmulqdq" }; -#endif /* defined(__x86_64) && defined(HAVE_PCLMULQDQ) */ +#endif /* defined(__x86_64) && HAVE_SIMD(PCLMULQDQ) */ diff --git a/module/icp/algs/sha2/sha256_impl.c b/module/icp/algs/sha2/sha256_impl.c index dcb0a391d..a6a4a0d9a 100644 --- a/module/icp/algs/sha2/sha256_impl.c +++ b/module/icp/algs/sha2/sha256_impl.c @@ -65,7 +65,7 @@ const sha256_ops_t sha256_x64_impl = { .name = "x64" }; -#if defined(HAVE_SSSE3) +#if HAVE_SIMD(SSSE3) static boolean_t sha2_have_ssse3(void) { return (kfpu_allowed() && zfs_ssse3_available()); @@ -79,7 +79,7 @@ const sha256_ops_t sha256_ssse3_impl = { }; #endif -#if defined(HAVE_AVX) +#if HAVE_SIMD(AVX) static boolean_t sha2_have_avx(void) { return (kfpu_allowed() && zfs_avx_available()); @@ -93,7 +93,7 @@ const sha256_ops_t sha256_avx_impl = { }; #endif -#if defined(HAVE_AVX2) +#if HAVE_SIMD(AVX2) static boolean_t sha2_have_avx2(void) { return (kfpu_allowed() && zfs_avx2_available()); @@ -107,7 +107,7 @@ const sha256_ops_t sha256_avx2_impl = { }; #endif -#if defined(HAVE_SSE4_1) +#if HAVE_SIMD(SSE4_1) static boolean_t sha2_have_shani(void) { return (kfpu_allowed() && zfs_sse4_1_available() && \ @@ -186,16 +186,16 @@ static const sha256_ops_t *const sha256_impls[] = { #if defined(__x86_64) &sha256_x64_impl, #endif -#if defined(__x86_64) && defined(HAVE_SSSE3) +#if defined(__x86_64) && HAVE_SIMD(SSSE3) &sha256_ssse3_impl, #endif -#if defined(__x86_64) && defined(HAVE_AVX) +#if defined(__x86_64) && HAVE_SIMD(AVX) &sha256_avx_impl, #endif -#if defined(__x86_64) && defined(HAVE_AVX2) +#if defined(__x86_64) && HAVE_SIMD(AVX2) &sha256_avx2_impl, #endif -#if defined(__x86_64) && defined(HAVE_SSE4_1) +#if defined(__x86_64) && HAVE_SIMD(SSE4_1) &sha256_shani_impl, #endif #if defined(__aarch64__) || defined(__arm__) diff --git a/module/icp/algs/sha2/sha512_impl.c b/module/icp/algs/sha2/sha512_impl.c index 4206f8f20..19f9a8cca 100644 --- a/module/icp/algs/sha2/sha512_impl.c +++ b/module/icp/algs/sha2/sha512_impl.c @@ -65,7 +65,7 @@ const sha512_ops_t sha512_x64_impl = { .name = "x64" }; -#if defined(HAVE_AVX) +#if HAVE_SIMD(AVX) static boolean_t sha2_have_avx(void) { return (kfpu_allowed() && zfs_avx_available()); @@ -79,7 +79,7 @@ const sha512_ops_t sha512_avx_impl = { }; #endif -#if defined(HAVE_AVX2) +#if HAVE_SIMD(AVX2) static boolean_t sha2_have_avx2(void) { return (kfpu_allowed() && zfs_avx2_available()); @@ -93,7 +93,7 @@ const sha512_ops_t sha512_avx2_impl = { }; #endif -#if defined(HAVE_SHA512EXT) +#if HAVE_SIMD(SHA512EXT) static boolean_t sha2_have_sha512ext(void) { return (kfpu_allowed() && zfs_sha512ext_available()); @@ -173,13 +173,13 @@ static const sha512_ops_t *const sha512_impls[] = { #if defined(__x86_64) &sha512_x64_impl, #endif -#if defined(__x86_64) && defined(HAVE_AVX) +#if defined(__x86_64) && HAVE_SIMD(AVX) &sha512_avx_impl, #endif -#if defined(__x86_64) && defined(HAVE_AVX2) +#if defined(__x86_64) && HAVE_SIMD(AVX2) &sha512_avx2_impl, #endif -#if defined(__x86_64) && defined(HAVE_SHA512EXT) +#if defined(__x86_64) && HAVE_SIMD(SHA512EXT) &sha512_sha512ext_impl, #endif #if defined(__aarch64__) || defined(__arm__) diff --git a/module/icp/asm-x86_64/aes/aes_aesni.S b/module/icp/asm-x86_64/aes/aes_aesni.S index b8ab93794..4a216274b 100644 --- a/module/icp/asm-x86_64/aes/aes_aesni.S +++ b/module/icp/asm-x86_64/aes/aes_aesni.S @@ -179,7 +179,7 @@ rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[], } -#elif defined(HAVE_AES) /* guard by instruction set */ +#elif HAVE_SIMD(AES) /* guard by instruction set */ #define _ASM #include diff --git a/module/icp/asm-x86_64/blake3/blake3_avx2.S b/module/icp/asm-x86_64/blake3/blake3_avx2.S index 6edd59e6e..3e52dfdad 100644 --- a/module/icp/asm-x86_64/blake3/blake3_avx2.S +++ b/module/icp/asm-x86_64/blake3/blake3_avx2.S @@ -26,7 +26,7 @@ * Copyright (c) 2022 Tino Reichardt */ -#if defined(HAVE_AVX2) +#if HAVE_SIMD(AVX2) #define _ASM #include @@ -1822,7 +1822,7 @@ CMP_MSB_MASK: .long 0x80000000, 0x80000000, 0x80000000, 0x80000000 BLAKE3_IV: .long 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A -#endif /* HAVE_AVX2 */ +#endif /* HAVE_SIMD(AVX2) */ #ifdef __ELF__ .section .note.GNU-stack,"",%progbits diff --git a/module/icp/asm-x86_64/blake3/blake3_avx512.S b/module/icp/asm-x86_64/blake3/blake3_avx512.S index 56af3e152..51a83bf35 100644 --- a/module/icp/asm-x86_64/blake3/blake3_avx512.S +++ b/module/icp/asm-x86_64/blake3/blake3_avx512.S @@ -26,7 +26,7 @@ * Copyright (c) 2022 Tino Reichardt */ -#if defined(HAVE_AVX512F) && defined(HAVE_AVX512VL) +#if HAVE_SIMD(AVX512F) && HAVE_SIMD(AVX512VL) #define _ASM #include @@ -2588,7 +2588,7 @@ BLAKE3_IV_2: BLAKE3_IV_3: .long 0xA54FF53A -#endif /* HAVE_AVX512 */ +#endif /* HAVE_SIMD(AVX512) */ #ifdef __ELF__ .section .note.GNU-stack,"",%progbits diff --git a/module/icp/asm-x86_64/blake3/blake3_sse2.S b/module/icp/asm-x86_64/blake3/blake3_sse2.S index 6d9c67206..42b5d81db 100644 --- a/module/icp/asm-x86_64/blake3/blake3_sse2.S +++ b/module/icp/asm-x86_64/blake3/blake3_sse2.S @@ -26,7 +26,7 @@ * Copyright (c) 2022 Tino Reichardt */ -#if defined(HAVE_SSE2) +#if HAVE_SIMD(SSE2) #define _ASM #include @@ -2293,7 +2293,7 @@ PBLENDW_0x3F_MASK: PBLENDW_0xC0_MASK: .long 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF -#endif /* HAVE_SSE2 */ +#endif /* HAVE_SIMD(SSE2) */ #ifdef __ELF__ .section .note.GNU-stack,"",%progbits diff --git a/module/icp/asm-x86_64/blake3/blake3_sse41.S b/module/icp/asm-x86_64/blake3/blake3_sse41.S index eb4609622..98caf0cd2 100644 --- a/module/icp/asm-x86_64/blake3/blake3_sse41.S +++ b/module/icp/asm-x86_64/blake3/blake3_sse41.S @@ -26,7 +26,7 @@ * Copyright (c) 2022 Tino Reichardt */ -#if defined(HAVE_SSE4_1) +#if HAVE_SIMD(SSE4_1) #define _ASM #include @@ -2031,7 +2031,7 @@ BLAKE3_BLOCK_LEN: CMP_MSB_MASK: .long 0x80000000, 0x80000000, 0x80000000, 0x80000000 -#endif /* HAVE_SSE4_1 */ +#endif /* HAVE_SIMD(SSE4_1) */ #ifdef __ELF__ .section .note.GNU-stack,"",%progbits diff --git a/module/icp/asm-x86_64/modes/aesni-gcm-avx2-vaes.S b/module/icp/asm-x86_64/modes/aesni-gcm-avx2-vaes.S index 626f4639c..bcba49b6d 100644 --- a/module/icp/asm-x86_64/modes/aesni-gcm-avx2-vaes.S +++ b/module/icp/asm-x86_64/modes/aesni-gcm-avx2-vaes.S @@ -5,8 +5,8 @@ // perlasm source: https://github.com/google/boringssl/blob/d5440dd2c2c500ac2d3bba4afec47a054b4d99ae/crypto/fipsmodule/aes/asm/aes-gcm-avx2-x86_64.pl // generated source: https://github.com/google/boringssl/blob/d5440dd2c2c500ac2d3bba4afec47a054b4d99ae/gen/bcm/aes-gcm-avx2-x86_64-linux.S -#if defined(__x86_64__) && defined(HAVE_AVX2) && \ - defined(HAVE_VAES) && defined(HAVE_VPCLMULQDQ) +#if defined(__x86_64__) && HAVE_SIMD(AVX2) && \ + HAVE_SIMD(VAES) && HAVE_SIMD(VPCLMULQDQ) #define _ASM #include @@ -1288,4 +1288,4 @@ SET_SIZE(aes_gcm_dec_update_vaes_avx2) .section .note.GNU-stack,"",%progbits #endif -#endif /* defined(__x86_64__) && defined(HAVE_AVX2) && defined(HAVE_VAES) ... */ +#endif /* defined(__x86_64__) && HAVE_SIMD(AVX2) && HAVE_SIMD(VAES) ... */ diff --git a/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S b/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S index 2e0a88ec8..9929f6af9 100644 --- a/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S +++ b/module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S @@ -45,8 +45,8 @@ # and modified for ICP. Modification are kept at a bare minimum to ease later # upstream merges. -#if defined(__x86_64__) && defined(HAVE_AVX) && \ - defined(HAVE_AES) && defined(HAVE_PCLMULQDQ) +#if defined(__x86_64__) && HAVE_SIMD(AVX) && \ + HAVE_SIMD(AES) && HAVE_SIMD(PCLMULQDQ) #define _ASM #include @@ -64,7 +64,7 @@ .text -#ifdef HAVE_MOVBE +#if HAVE_SIMD(MOVBE) .balign 32 FUNCTION(_aesni_ctr32_ghash_6x) .cfi_startproc @@ -380,7 +380,7 @@ FUNCTION(_aesni_ctr32_ghash_6x) .cfi_endproc SET_SIZE(_aesni_ctr32_ghash_6x) STACK_FRAME_NON_STANDARD _aesni_ctr32_ghash_6x -#endif /* ifdef HAVE_MOVBE */ +#endif /* HAVE_SIMD(MOVBE) */ .balign 32 FUNCTION(_aesni_ctr32_ghash_no_movbe_6x) @@ -781,7 +781,7 @@ ENTRY_ALIGN(aesni_gcm_decrypt, 32) vmovdqu %xmm2,96(%rsp) vmovdqu %xmm3,112(%rsp) -#ifdef HAVE_MOVBE +#if HAVE_SIMD(MOVBE) #ifdef _KERNEL testl $1,gcm_avx_can_use_movbe(%rip) #else @@ -992,7 +992,7 @@ ENTRY_ALIGN(aesni_gcm_encrypt, 32) movq $192,%r10 vpshufb %xmm0,%xmm8,%xmm8 -#ifdef HAVE_MOVBE +#if HAVE_SIMD(MOVBE) #ifdef _KERNEL testl $1,gcm_avx_can_use_movbe(%rip) #else @@ -1266,7 +1266,7 @@ SECTION_STATIC #if defined(__linux__) && defined(HAVE_STACK_FRAME_NON_STANDARD) && \ ! defined(HAVE_STACK_FRAME_NON_STANDARD_ASM) .section .discard.func_stack_frame_non_standard, "aw" -#ifdef HAVE_MOVBE +#if HAVE_SIMD(MOVBE) .long _aesni_ctr32_ghash_6x - . #endif .long _aesni_ctr32_ghash_no_movbe_6x - . @@ -1279,4 +1279,4 @@ SECTION_STATIC .section .note.GNU-stack,"",%progbits #endif -#endif /* defined(__x86_64__) && defined(HAVE_AVX) && defined(HAVE_AES) ... */ +#endif /* defined(__x86_64__) && HAVE_SIMD(AVX) && HAVE_SIMD(AES) ... */ diff --git a/module/icp/asm-x86_64/modes/gcm_pclmulqdq.S b/module/icp/asm-x86_64/modes/gcm_pclmulqdq.S index f5d3b3362..82eac5b48 100644 --- a/module/icp/asm-x86_64/modes/gcm_pclmulqdq.S +++ b/module/icp/asm-x86_64/modes/gcm_pclmulqdq.S @@ -91,7 +91,7 @@ gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res) { (void) x_in, (void) y, (void) res; } -#elif defined(HAVE_PCLMULQDQ) /* guard by instruction set */ +#elif HAVE_SIMD(PCLMULQDQ) /* guard by instruction set */ #define _ASM #include diff --git a/module/icp/asm-x86_64/modes/ghash-x86_64.S b/module/icp/asm-x86_64/modes/ghash-x86_64.S index 8295f91d2..9027684a2 100644 --- a/module/icp/asm-x86_64/modes/ghash-x86_64.S +++ b/module/icp/asm-x86_64/modes/ghash-x86_64.S @@ -95,8 +95,8 @@ # and modified for ICP. Modification are kept at a bare minimum to ease later # upstream merges. -#if defined(__x86_64__) && defined(HAVE_AVX) && \ - defined(HAVE_AES) && defined(HAVE_PCLMULQDQ) +#if defined(__x86_64__) && HAVE_SIMD(AVX) && \ + HAVE_SIMD(AES) && HAVE_SIMD(PCLMULQDQ) #define _ASM #include @@ -654,4 +654,4 @@ SET_OBJ(.Lrem_8bit) .section .note.GNU-stack,"",%progbits #endif -#endif /* defined(__x86_64__) && defined(HAVE_AVX) && defined(HAVE_AES) ... */ +#endif /* defined(__x86_64__) && HAVE_SIMD(AVX) && HAVE_SIMD(AES) ... */ diff --git a/module/icp/asm-x86_64/sha2/sha512-x86_64.S b/module/icp/asm-x86_64/sha2/sha512-x86_64.S index 47e4edd51..c281bc64d 100644 --- a/module/icp/asm-x86_64/sha2/sha512-x86_64.S +++ b/module/icp/asm-x86_64/sha2/sha512-x86_64.S @@ -4054,7 +4054,7 @@ ENTRY_ALIGN(zfs_sha512_transform_avx2, 64) SET_SIZE(zfs_sha512_transform_avx2) STACK_FRAME_NON_STANDARD zfs_sha512_transform_avx2 -#ifdef HAVE_SHA512EXT +#if HAVE_SIMD(SHA512EXT) ENTRY_ALIGN(zfs_sha512_transform_sha512ext, 64) .cfi_startproc ENDBR @@ -4324,7 +4324,7 @@ ENTRY_ALIGN(zfs_sha512_transform_sha512ext, 64) .cfi_endproc SET_SIZE(zfs_sha512_transform_sha512ext) STACK_FRAME_NON_STANDARD zfs_sha512_transform_sha512ext -#endif /* HAVE_SHA512EXT */ +#endif /* HAVE_SIMD(SHA512EXT) */ /* Workaround for missing asm macro in RHEL 8. */ #if defined(__linux__) && defined(HAVE_STACK_FRAME_NON_STANDARD) && \ @@ -4333,7 +4333,7 @@ STACK_FRAME_NON_STANDARD zfs_sha512_transform_sha512ext .long zfs_sha512_transform_x64 - . .long zfs_sha512_transform_avx - . .long zfs_sha512_transform_avx2 - . -#ifdef HAVE_SHA512EXT +#if HAVE_SIMD(SHA512EXT) .long zfs_sha512_transform_sha512ext - . #endif #endif diff --git a/module/icp/include/aes/aes_impl.h b/module/icp/include/aes/aes_impl.h index 17cbbac18..4b7d13312 100644 --- a/module/icp/include/aes/aes_impl.h +++ b/module/icp/include/aes/aes_impl.h @@ -196,7 +196,7 @@ extern ASMABI void aes_encrypt_amd64(const uint32_t rk[], int Nr, extern ASMABI void aes_decrypt_amd64(const uint32_t rk[], int Nr, const uint32_t ct[4], uint32_t pt[4]); #endif -#if defined(__x86_64) && defined(HAVE_AES) +#if defined(__x86_64) && HAVE_SIMD(AES) extern const aes_impl_ops_t aes_aesni_impl; #endif diff --git a/module/icp/include/modes/gcm_impl.h b/module/icp/include/modes/gcm_impl.h index 238a4e27f..0bc1567da 100644 --- a/module/icp/include/modes/gcm_impl.h +++ b/module/icp/include/modes/gcm_impl.h @@ -55,7 +55,7 @@ typedef struct gcm_impl_ops { } gcm_impl_ops_t; extern const gcm_impl_ops_t gcm_generic_impl; -#if defined(__x86_64) && defined(HAVE_PCLMULQDQ) +#if defined(__x86_64) && HAVE_SIMD(PCLMULQDQ) extern const gcm_impl_ops_t gcm_pclmulqdq_impl; #endif diff --git a/module/icp/include/modes/modes.h b/module/icp/include/modes/modes.h index de11d9eaf..66fa11712 100644 --- a/module/icp/include/modes/modes.h +++ b/module/icp/include/modes/modes.h @@ -40,9 +40,9 @@ extern "C" { * routines. AVX support should imply AES-NI and PCLMULQDQ, but make sure * anyhow. */ -#if defined(__x86_64__) && defined(HAVE_AVX) && \ - defined(HAVE_AES) && defined(HAVE_PCLMULQDQ) -#define CAN_USE_GCM_ASM (HAVE_VAES && HAVE_VPCLMULQDQ ? 2 : 1) +#if defined(__x86_64__) && HAVE_SIMD(AVX) && \ + HAVE_SIMD(AES) && HAVE_SIMD(PCLMULQDQ) +#define CAN_USE_GCM_ASM (HAVE_SIMD(VAES) && HAVE_SIMD(VPCLMULQDQ) ? 2 : 1) extern boolean_t gcm_avx_can_use_movbe; #endif diff --git a/module/zcommon/zfs_fletcher.c b/module/zcommon/zfs_fletcher.c index 133217f4b..1a7dde1dd 100644 --- a/module/zcommon/zfs_fletcher.c +++ b/module/zcommon/zfs_fletcher.c @@ -174,19 +174,19 @@ static const fletcher_4_ops_t *fletcher_4_impls[] = { &fletcher_4_scalar_ops, &fletcher_4_superscalar_ops, &fletcher_4_superscalar4_ops, -#if defined(HAVE_SSE2) +#if HAVE_SIMD(SSE2) &fletcher_4_sse2_ops, #endif -#if defined(HAVE_SSE2) && defined(HAVE_SSSE3) +#if HAVE_SIMD(SSE2) && HAVE_SIMD(SSSE3) &fletcher_4_ssse3_ops, #endif -#if defined(HAVE_AVX) && defined(HAVE_AVX2) +#if HAVE_SIMD(AVX) && HAVE_SIMD(AVX2) &fletcher_4_avx2_ops, #endif -#if defined(__x86_64) && defined(HAVE_AVX512F) +#if defined(__x86_64) && HAVE_SIMD(AVX512F) &fletcher_4_avx512f_ops, #endif -#if defined(__x86_64) && defined(HAVE_AVX512BW) +#if defined(__x86_64) && HAVE_SIMD(AVX512BW) &fletcher_4_avx512bw_ops, #endif #if defined(__aarch64__) && !defined(__FreeBSD__) diff --git a/module/zcommon/zfs_fletcher_avx512.c b/module/zcommon/zfs_fletcher_avx512.c index b42ebee57..ac3667d30 100644 --- a/module/zcommon/zfs_fletcher_avx512.c +++ b/module/zcommon/zfs_fletcher_avx512.c @@ -23,7 +23,7 @@ * Copyright (C) 2016 Gvozden Nešković. All rights reserved. */ -#if defined(__x86_64) && defined(HAVE_AVX512F) +#if defined(__x86_64) && HAVE_SIMD(AVX512F) #include #include @@ -167,7 +167,7 @@ const fletcher_4_ops_t fletcher_4_avx512f_ops = { .name = "avx512f" }; -#if defined(HAVE_AVX512BW) +#if HAVE_SIMD(AVX512BW) static void fletcher_4_avx512bw_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) @@ -219,4 +219,4 @@ const fletcher_4_ops_t fletcher_4_avx512bw_ops = { }; #endif -#endif /* defined(__x86_64) && defined(HAVE_AVX512F) */ +#endif /* defined(__x86_64) && HAVE_SIMD(AVX512F) */ diff --git a/module/zcommon/zfs_fletcher_intel.c b/module/zcommon/zfs_fletcher_intel.c index 570076ba2..7da1dc4ee 100644 --- a/module/zcommon/zfs_fletcher_intel.c +++ b/module/zcommon/zfs_fletcher_intel.c @@ -41,7 +41,7 @@ * SOFTWARE. */ -#if defined(HAVE_AVX) && defined(HAVE_AVX2) +#if HAVE_SIMD(AVX) && HAVE_SIMD(AVX2) #include #include @@ -164,4 +164,4 @@ const fletcher_4_ops_t fletcher_4_avx2_ops = { .name = "avx2" }; -#endif /* defined(HAVE_AVX) && defined(HAVE_AVX2) */ +#endif /* HAVE_SIMD(AVX) && HAVE_SIMD(AVX2) */ diff --git a/module/zcommon/zfs_fletcher_sse.c b/module/zcommon/zfs_fletcher_sse.c index 96aaf1622..a656790c6 100644 --- a/module/zcommon/zfs_fletcher_sse.c +++ b/module/zcommon/zfs_fletcher_sse.c @@ -42,7 +42,7 @@ * SOFTWARE. */ -#if defined(HAVE_SSE2) +#if HAVE_SIMD(SSE2) #include #include @@ -165,9 +165,9 @@ const fletcher_4_ops_t fletcher_4_sse2_ops = { .name = "sse2" }; -#endif /* defined(HAVE_SSE2) */ +#endif /* HAVE_SIMD(SSE2) */ -#if defined(HAVE_SSE2) && defined(HAVE_SSSE3) +#if HAVE_SIMD(SSE2) && HAVE_SIMD(SSSE3) static void fletcher_4_ssse3_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) { @@ -220,4 +220,4 @@ const fletcher_4_ops_t fletcher_4_ssse3_ops = { .name = "ssse3" }; -#endif /* defined(HAVE_SSE2) && defined(HAVE_SSSE3) */ +#endif /* HAVE_SIMD(SSE2) && HAVE_SIMD(SSSE3) */ diff --git a/module/zfs/vdev_raidz_math.c b/module/zfs/vdev_raidz_math.c index bdc159d8e..ca8ead23e 100644 --- a/module/zfs/vdev_raidz_math.c +++ b/module/zfs/vdev_raidz_math.c @@ -47,19 +47,19 @@ static raidz_impl_ops_t vdev_raidz_fastest_impl = { static const raidz_impl_ops_t *const raidz_all_maths[] = { &vdev_raidz_original_impl, &vdev_raidz_scalar_impl, -#if defined(__x86_64) && defined(HAVE_SSE2) /* only x86_64 for now */ +#if defined(__x86_64) && HAVE_SIMD(SSE2) /* only x86_64 for now */ &vdev_raidz_sse2_impl, #endif -#if defined(__x86_64) && defined(HAVE_SSSE3) /* only x86_64 for now */ +#if defined(__x86_64) && HAVE_SIMD(SSSE3) /* only x86_64 for now */ &vdev_raidz_ssse3_impl, #endif -#if defined(__x86_64) && defined(HAVE_AVX2) /* only x86_64 for now */ +#if defined(__x86_64) && HAVE_SIMD(AVX2) /* only x86_64 for now */ &vdev_raidz_avx2_impl, #endif -#if defined(__x86_64) && defined(HAVE_AVX512F) /* only x86_64 for now */ +#if defined(__x86_64) && HAVE_SIMD(AVX512F) /* only x86_64 for now */ &vdev_raidz_avx512f_impl, #endif -#if defined(__x86_64) && defined(HAVE_AVX512BW) /* only x86_64 for now */ +#if defined(__x86_64) && HAVE_SIMD(AVX512BW) /* only x86_64 for now */ &vdev_raidz_avx512bw_impl, #endif #if defined(__aarch64__) && !defined(__FreeBSD__) diff --git a/module/zfs/vdev_raidz_math_avx2.c b/module/zfs/vdev_raidz_math_avx2.c index d0311e522..03444ccdf 100644 --- a/module/zfs/vdev_raidz_math_avx2.c +++ b/module/zfs/vdev_raidz_math_avx2.c @@ -22,9 +22,10 @@ /* * Copyright (C) 2016 Gvozden Nešković. All rights reserved. */ + #include -#if defined(__x86_64) && defined(HAVE_AVX2) +#if defined(__x86_64) && HAVE_SIMD(AVX2) #include #include @@ -411,4 +412,4 @@ const raidz_impl_ops_t vdev_raidz_avx2_impl = { .name = "avx2" }; -#endif /* defined(__x86_64) && defined(HAVE_AVX2) */ +#endif /* defined(__x86_64) && HAVE_SIMD(AVX2) */ diff --git a/module/zfs/vdev_raidz_math_avx512bw.c b/module/zfs/vdev_raidz_math_avx512bw.c index 5ad2915e3..0d9ed3938 100644 --- a/module/zfs/vdev_raidz_math_avx512bw.c +++ b/module/zfs/vdev_raidz_math_avx512bw.c @@ -26,7 +26,7 @@ #include -#if defined(__x86_64) && defined(HAVE_AVX512BW) +#if defined(__x86_64) && HAVE_SIMD(AVX512BW) #include #include @@ -411,4 +411,4 @@ const raidz_impl_ops_t vdev_raidz_avx512bw_impl = { .name = "avx512bw" }; -#endif /* defined(__x86_64) && defined(HAVE_AVX512BW) */ +#endif /* defined(__x86_64) && HAVE_SIMD(AVX512BW) */ diff --git a/module/zfs/vdev_raidz_math_avx512f.c b/module/zfs/vdev_raidz_math_avx512f.c index 17eb9f709..e7738ff41 100644 --- a/module/zfs/vdev_raidz_math_avx512f.c +++ b/module/zfs/vdev_raidz_math_avx512f.c @@ -26,7 +26,7 @@ #include -#if defined(__x86_64) && defined(HAVE_AVX512F) +#if defined(__x86_64) && HAVE_SIMD(AVX512F) #include #include @@ -492,4 +492,4 @@ const raidz_impl_ops_t vdev_raidz_avx512f_impl = { .name = "avx512f" }; -#endif /* defined(__x86_64) && defined(HAVE_AVX512F) */ +#endif /* defined(__x86_64) && HAVE_SIMD(AVX512F) */ diff --git a/module/zfs/vdev_raidz_math_sse2.c b/module/zfs/vdev_raidz_math_sse2.c index 3be4edeaa..05037881a 100644 --- a/module/zfs/vdev_raidz_math_sse2.c +++ b/module/zfs/vdev_raidz_math_sse2.c @@ -25,7 +25,7 @@ #include -#if defined(__x86_64) && defined(HAVE_SSE2) +#if defined(__x86_64) && HAVE_SIMD(SSE2) #include #include @@ -629,4 +629,4 @@ const raidz_impl_ops_t vdev_raidz_sse2_impl = { .name = "sse2" }; -#endif /* defined(__x86_64) && defined(HAVE_SSE2) */ +#endif /* defined(__x86_64) && HAVE_SIMD(SSE2) */ diff --git a/module/zfs/vdev_raidz_math_ssse3.c b/module/zfs/vdev_raidz_math_ssse3.c index 4067aaead..88dcde379 100644 --- a/module/zfs/vdev_raidz_math_ssse3.c +++ b/module/zfs/vdev_raidz_math_ssse3.c @@ -25,7 +25,7 @@ #include -#if defined(__x86_64) && defined(HAVE_SSSE3) +#if defined(__x86_64) && HAVE_SIMD(SSSE3) #include #include @@ -415,11 +415,11 @@ const raidz_impl_ops_t vdev_raidz_ssse3_impl = { .name = "ssse3" }; -#endif /* defined(__x86_64) && defined(HAVE_SSSE3) */ +#endif /* defined(__x86_64) && HAVE_SIMD(SSSE3) */ #if defined(__x86_64) -#if defined(HAVE_SSSE3) || defined(HAVE_AVX2) || defined(HAVE_AVX512BW) +#if HAVE_SIMD(SSSE3) || HAVE_SIMD(AVX2) || HAVE_SIMD(AVX512BW) /* BEGIN CSTYLED */ const uint8_t __attribute__((aligned(256))) gf_clmul_mod_lt[4*256][16] = @@ -2474,5 +2474,5 @@ __attribute__((aligned(256))) gf_clmul_mod_lt[4*256][16] = 0xf8, 0x07, 0x06, 0xf9, 0x04, 0xfb, 0xfa, 0x05 } }; /* END CSTYLED */ -#endif /* defined(HAVE_SSSE3) || defined(HAVE_AVX2) || defined(HAVE_AVX512BW) */ +#endif /* HAVE_SIMD(SSSE3) || HAVE_SIMD(AVX2) || HAVE_SIMD(AVX512BW) */ #endif /* defined(__x86_64) */