c/mirror_zfs
1
0
Fork 0
mirror of https://git.proxmox.com/git/mirror_zfs.git synced 2026-05-22 10:37:35 +03:00
Code Issues Packages Projects Releases Wiki Activity

Convert all HAVE_<name> SIMD gates to HAVE_SIMD(<name>)

Browse Source 
The original names no longer exist, and the new ones will need to be
selectable based on the current compilation target.

Sponsored-by: TrueNAS
Reviewed-by: Brian Behlendorf <behlendorf1@llnl.gov>
Signed-off-by: Rob Norris <rob.norris@truenas.com>
Closes #18285
This commit is contained in:
Rob Norris
2026-03-04 22:28:21 +11:00
committed by Brian Behlendorf
parent 92a6ab405f
commit 35f74f84e6
33 changed files with 113 additions and 112 deletions
Download Patch File Download Diff File Expand all files Collapse all files
module/icp/algs/aes/aes_impl.c
+2 -2
View File
@@ -231,7 +231,7 @@ static const aes_impl_ops_t *aes_all_impl[] = {
#if defined(__x86_64)
&aes_x86_64_impl,
#endif
#if defined(__x86_64) && defined(HAVE_AES)
#if defined(__x86_64) && HAVE_SIMD(AES)
&aes_aesni_impl,
#endif
};
@@ -315,7 +315,7 @@ aes_impl_init(void)
* hardware accelerated version is the fastest.
*/
#if defined(__x86_64)
#if defined(HAVE_AES)
#if HAVE_SIMD(AES)
if (aes_aesni_impl.is_supported()) {
memcpy(&aes_fastest_impl, &aes_aesni_impl,
sizeof (aes_fastest_impl));
module/icp/algs/aes/aes_impl_aesni.c
+2 -2
View File
@@ -23,7 +23,7 @@
* Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
*/
#if defined(__x86_64) && defined(HAVE_AES)
#if defined(__x86_64) && HAVE_SIMD(AES)
#include <sys/simd.h>
#include <sys/types.h>
@@ -123,4 +123,4 @@ const aes_impl_ops_t aes_aesni_impl = {
.name = "aesni"
};
#endif /* defined(__x86_64) && defined(HAVE_AES) */
#endif /* defined(__x86_64) && HAVE_SIMD(AES) */
module/icp/algs/blake3/blake3_impl.c
+8 -8
View File
@@ -32,7 +32,7 @@
#include "blake3_impl.h"
#if defined(__aarch64__) || \
(defined(__x86_64) && defined(HAVE_SSE2)) || \
(defined(__x86_64) && HAVE_SIMD(SSE2)) || \
(defined(__PPC64__) && defined(__LITTLE_ENDIAN__))
extern void ASMABI zfs_blake3_compress_in_place_sse2(uint32_t cv[8],
@@ -98,7 +98,7 @@ const blake3_ops_t blake3_sse2_impl = {
#endif
#if defined(__aarch64__) || \
(defined(__x86_64) && defined(HAVE_SSE2)) || \
(defined(__x86_64) && HAVE_SIMD(SSE2)) || \
(defined(__PPC64__) && defined(__LITTLE_ENDIAN__))
extern void ASMABI zfs_blake3_compress_in_place_sse41(uint32_t cv[8],
@@ -163,7 +163,7 @@ const blake3_ops_t blake3_sse41_impl = {
};
#endif
#if defined(__x86_64) && defined(HAVE_SSE4_1) && defined(HAVE_AVX2)
#if defined(__x86_64) && HAVE_SIMD(SSE4_1) && HAVE_SIMD(AVX2)
extern void ASMABI zfs_blake3_hash_many_avx2(const uint8_t * const *inputs,
size_t num_inputs, size_t blocks, const uint32_t key[8],
uint64_t counter, boolean_t increment_counter, uint8_t flags,
@@ -196,7 +196,7 @@ blake3_avx2_impl = {
};
#endif
#if defined(__x86_64) && defined(HAVE_AVX512F) && defined(HAVE_AVX512VL)
#if defined(__x86_64) && HAVE_SIMD(AVX512F) && HAVE_SIMD(AVX512VL)
extern void ASMABI zfs_blake3_compress_in_place_avx512(uint32_t cv[8],
const uint8_t block[BLAKE3_BLOCK_LEN], uint8_t block_len,
uint64_t counter, uint8_t flags);
@@ -259,19 +259,19 @@ extern const blake3_ops_t blake3_generic_impl;
static const blake3_ops_t *const blake3_impls[] = {
&blake3_generic_impl,
#if defined(__aarch64__) || \
(defined(__x86_64) && defined(HAVE_SSE2)) || \
(defined(__x86_64) && HAVE_SIMD(SSE2)) || \
(defined(__PPC64__) && defined(__LITTLE_ENDIAN__))
&blake3_sse2_impl,
#endif
#if defined(__aarch64__) || \
(defined(__x86_64) && defined(HAVE_SSE4_1)) || \
(defined(__x86_64) && HAVE_SIMD(SSE4_1)) || \
(defined(__PPC64__) && defined(__LITTLE_ENDIAN__))
&blake3_sse41_impl,
#endif
#if defined(__x86_64) && defined(HAVE_SSE4_1) && defined(HAVE_AVX2)
#if defined(__x86_64) && HAVE_SIMD(SSE4_1) && HAVE_SIMD(AVX2)
&blake3_avx2_impl,
#endif
#if defined(__x86_64) && defined(HAVE_AVX512F) && defined(HAVE_AVX512VL)
#if defined(__x86_64) && HAVE_SIMD(AVX512F) && HAVE_SIMD(AVX512VL)
&blake3_avx512_impl,
#endif
};
module/icp/algs/modes/gcm.c
+3 -3
View File
@@ -714,7 +714,7 @@ static gcm_impl_ops_t gcm_fastest_impl = {
/* All compiled in implementations */
static const gcm_impl_ops_t *gcm_all_impl[] = {
&gcm_generic_impl,
#if defined(__x86_64) && defined(HAVE_PCLMULQDQ)
#if defined(__x86_64) && HAVE_SIMD(PCLMULQDQ)
&gcm_pclmulqdq_impl,
#endif
};
@@ -801,7 +801,7 @@ gcm_impl_init(void)
* Set the fastest implementation given the assumption that the
* hardware accelerated version is the fastest.
*/
#if defined(__x86_64) && defined(HAVE_PCLMULQDQ)
#if defined(__x86_64) && HAVE_SIMD(PCLMULQDQ)
if (gcm_pclmulqdq_impl.is_supported()) {
memcpy(&gcm_fastest_impl, &gcm_pclmulqdq_impl,
sizeof (gcm_fastest_impl));
@@ -827,7 +827,7 @@ gcm_impl_init(void)
} else
#endif
if (gcm_avx_will_work()) {
#ifdef HAVE_MOVBE
#if HAVE_SIMD(MOVBE)
if (zfs_movbe_available() == B_TRUE) {
atomic_swap_32(&gcm_avx_can_use_movbe, B_TRUE);
}
module/icp/algs/modes/gcm_pclmulqdq.c
+2 -2
View File
@@ -23,7 +23,7 @@
* Copyright (c) 2003, 2010, Oracle and/or its affiliates. All rights reserved.
*/
#if defined(__x86_64) && defined(HAVE_PCLMULQDQ)
#if defined(__x86_64) && HAVE_SIMD(PCLMULQDQ)
#include <sys/types.h>
#include <sys/simd.h>
@@ -63,4 +63,4 @@ const gcm_impl_ops_t gcm_pclmulqdq_impl = {
.name = "pclmulqdq"
};
#endif /* defined(__x86_64) && defined(HAVE_PCLMULQDQ) */
#endif /* defined(__x86_64) && HAVE_SIMD(PCLMULQDQ) */
module/icp/algs/sha2/sha256_impl.c
+8 -8
View File
@@ -65,7 +65,7 @@ const sha256_ops_t sha256_x64_impl = {
.name = "x64"
};
#if defined(HAVE_SSSE3)
#if HAVE_SIMD(SSSE3)
static boolean_t sha2_have_ssse3(void)
{
return (kfpu_allowed() && zfs_ssse3_available());
@@ -79,7 +79,7 @@ const sha256_ops_t sha256_ssse3_impl = {
};
#endif
#if defined(HAVE_AVX)
#if HAVE_SIMD(AVX)
static boolean_t sha2_have_avx(void)
{
return (kfpu_allowed() && zfs_avx_available());
@@ -93,7 +93,7 @@ const sha256_ops_t sha256_avx_impl = {
};
#endif
#if defined(HAVE_AVX2)
#if HAVE_SIMD(AVX2)
static boolean_t sha2_have_avx2(void)
{
return (kfpu_allowed() && zfs_avx2_available());
@@ -107,7 +107,7 @@ const sha256_ops_t sha256_avx2_impl = {
};
#endif
#if defined(HAVE_SSE4_1)
#if HAVE_SIMD(SSE4_1)
static boolean_t sha2_have_shani(void)
{
return (kfpu_allowed() && zfs_sse4_1_available() && \
@@ -186,16 +186,16 @@ static const sha256_ops_t *const sha256_impls[] = {
#if defined(__x86_64)
&sha256_x64_impl,
#endif
#if defined(__x86_64) && defined(HAVE_SSSE3)
#if defined(__x86_64) && HAVE_SIMD(SSSE3)
&sha256_ssse3_impl,
#endif
#if defined(__x86_64) && defined(HAVE_AVX)
#if defined(__x86_64) && HAVE_SIMD(AVX)
&sha256_avx_impl,
#endif
#if defined(__x86_64) && defined(HAVE_AVX2)
#if defined(__x86_64) && HAVE_SIMD(AVX2)
&sha256_avx2_impl,
#endif
#if defined(__x86_64) && defined(HAVE_SSE4_1)
#if defined(__x86_64) && HAVE_SIMD(SSE4_1)
&sha256_shani_impl,
#endif
#if defined(__aarch64__) || defined(__arm__)
module/icp/algs/sha2/sha512_impl.c
+6 -6
View File
@@ -65,7 +65,7 @@ const sha512_ops_t sha512_x64_impl = {
.name = "x64"
};
#if defined(HAVE_AVX)
#if HAVE_SIMD(AVX)
static boolean_t sha2_have_avx(void)
{
return (kfpu_allowed() && zfs_avx_available());
@@ -79,7 +79,7 @@ const sha512_ops_t sha512_avx_impl = {
};
#endif
#if defined(HAVE_AVX2)
#if HAVE_SIMD(AVX2)
static boolean_t sha2_have_avx2(void)
{
return (kfpu_allowed() && zfs_avx2_available());
@@ -93,7 +93,7 @@ const sha512_ops_t sha512_avx2_impl = {
};
#endif
#if defined(HAVE_SHA512EXT)
#if HAVE_SIMD(SHA512EXT)
static boolean_t sha2_have_sha512ext(void)
{
return (kfpu_allowed() && zfs_sha512ext_available());
@@ -173,13 +173,13 @@ static const sha512_ops_t *const sha512_impls[] = {
#if defined(__x86_64)
&sha512_x64_impl,
#endif
#if defined(__x86_64) && defined(HAVE_AVX)
#if defined(__x86_64) && HAVE_SIMD(AVX)
&sha512_avx_impl,
#endif
#if defined(__x86_64) && defined(HAVE_AVX2)
#if defined(__x86_64) && HAVE_SIMD(AVX2)
&sha512_avx2_impl,
#endif
#if defined(__x86_64) && defined(HAVE_SHA512EXT)
#if defined(__x86_64) && HAVE_SIMD(SHA512EXT)
&sha512_sha512ext_impl,
#endif
#if defined(__aarch64__) || defined(__arm__)
module/icp/asm-x86_64/aes/aes_aesni.S
+1 -1
View File
@@ -179,7 +179,7 @@ rijndael_key_setup_dec_intel(uint32_t rk[], const uint32_t cipherKey[],
}
#elif defined(HAVE_AES) /* guard by instruction set */
#elif HAVE_SIMD(AES) /* guard by instruction set */
#define _ASM
#include <sys/asm_linkage.h>
module/icp/asm-x86_64/blake3/blake3_avx2.S
+2 -2
View File
@@ -26,7 +26,7 @@
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
*/
#if defined(HAVE_AVX2)
#if HAVE_SIMD(AVX2)
#define _ASM
#include <sys/asm_linkage.h>
@@ -1822,7 +1822,7 @@ CMP_MSB_MASK:
.long 0x80000000, 0x80000000, 0x80000000, 0x80000000
BLAKE3_IV:
.long 0x6A09E667, 0xBB67AE85, 0x3C6EF372, 0xA54FF53A
#endif /* HAVE_AVX2 */
#endif /* HAVE_SIMD(AVX2) */
#ifdef __ELF__
.section .note.GNU-stack,"",%progbits
module/icp/asm-x86_64/blake3/blake3_avx512.S
+2 -2
View File
@@ -26,7 +26,7 @@
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
*/
#if defined(HAVE_AVX512F) && defined(HAVE_AVX512VL)
#if HAVE_SIMD(AVX512F) && HAVE_SIMD(AVX512VL)
#define _ASM
#include <sys/asm_linkage.h>
@@ -2588,7 +2588,7 @@ BLAKE3_IV_2:
BLAKE3_IV_3:
.long 0xA54FF53A
#endif /* HAVE_AVX512 */
#endif /* HAVE_SIMD(AVX512) */
#ifdef __ELF__
.section .note.GNU-stack,"",%progbits
module/icp/asm-x86_64/blake3/blake3_sse2.S
+2 -2
View File
@@ -26,7 +26,7 @@
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
*/
#if defined(HAVE_SSE2)
#if HAVE_SIMD(SSE2)
#define _ASM
#include <sys/asm_linkage.h>
@@ -2293,7 +2293,7 @@ PBLENDW_0x3F_MASK:
PBLENDW_0xC0_MASK:
.long 0x00000000, 0x00000000, 0x00000000, 0xFFFFFFFF
#endif /* HAVE_SSE2 */
#endif /* HAVE_SIMD(SSE2) */
#ifdef __ELF__
.section .note.GNU-stack,"",%progbits
module/icp/asm-x86_64/blake3/blake3_sse41.S
+2 -2
View File
@@ -26,7 +26,7 @@
* Copyright (c) 2022 Tino Reichardt <milky-zfs@mcmilk.de>
*/
#if defined(HAVE_SSE4_1)
#if HAVE_SIMD(SSE4_1)
#define _ASM
#include <sys/asm_linkage.h>
@@ -2031,7 +2031,7 @@ BLAKE3_BLOCK_LEN:
CMP_MSB_MASK:
.long 0x80000000, 0x80000000, 0x80000000, 0x80000000
#endif /* HAVE_SSE4_1 */
#endif /* HAVE_SIMD(SSE4_1) */
#ifdef __ELF__
.section .note.GNU-stack,"",%progbits
module/icp/asm-x86_64/modes/aesni-gcm-avx2-vaes.S
+3 -3
View File
@@ -5,8 +5,8 @@
// perlasm source: https://github.com/google/boringssl/blob/d5440dd2c2c500ac2d3bba4afec47a054b4d99ae/crypto/fipsmodule/aes/asm/aes-gcm-avx2-x86_64.pl
// generated source: https://github.com/google/boringssl/blob/d5440dd2c2c500ac2d3bba4afec47a054b4d99ae/gen/bcm/aes-gcm-avx2-x86_64-linux.S
#if defined(__x86_64__) && defined(HAVE_AVX2) && \
defined(HAVE_VAES) && defined(HAVE_VPCLMULQDQ)
#if defined(__x86_64__) && HAVE_SIMD(AVX2) && \
HAVE_SIMD(VAES) && HAVE_SIMD(VPCLMULQDQ)
#define _ASM
#include <sys/asm_linkage.h>
@@ -1288,4 +1288,4 @@ SET_SIZE(aes_gcm_dec_update_vaes_avx2)
.section .note.GNU-stack,"",%progbits
#endif
#endif /* defined(__x86_64__) && defined(HAVE_AVX2) && defined(HAVE_VAES) ... */
#endif /* defined(__x86_64__) && HAVE_SIMD(AVX2) && HAVE_SIMD(VAES) ... */
module/icp/asm-x86_64/modes/aesni-gcm-x86_64.S
+8 -8
View File
@@ -45,8 +45,8 @@
# and modified for ICP. Modification are kept at a bare minimum to ease later
# upstream merges.
#if defined(__x86_64__) && defined(HAVE_AVX) && \
defined(HAVE_AES) && defined(HAVE_PCLMULQDQ)
#if defined(__x86_64__) && HAVE_SIMD(AVX) && \
HAVE_SIMD(AES) && HAVE_SIMD(PCLMULQDQ)
#define _ASM
#include <sys/asm_linkage.h>
@@ -64,7 +64,7 @@
.text
#ifdef HAVE_MOVBE
#if HAVE_SIMD(MOVBE)
.balign 32
FUNCTION(_aesni_ctr32_ghash_6x)
.cfi_startproc
@@ -380,7 +380,7 @@ FUNCTION(_aesni_ctr32_ghash_6x)
.cfi_endproc
SET_SIZE(_aesni_ctr32_ghash_6x)
STACK_FRAME_NON_STANDARD _aesni_ctr32_ghash_6x
#endif /* ifdef HAVE_MOVBE */
#endif /* HAVE_SIMD(MOVBE) */
.balign 32
FUNCTION(_aesni_ctr32_ghash_no_movbe_6x)
@@ -781,7 +781,7 @@ ENTRY_ALIGN(aesni_gcm_decrypt, 32)
vmovdqu %xmm2,96(%rsp)
vmovdqu %xmm3,112(%rsp)
#ifdef HAVE_MOVBE
#if HAVE_SIMD(MOVBE)
#ifdef _KERNEL
testl $1,gcm_avx_can_use_movbe(%rip)
#else
@@ -992,7 +992,7 @@ ENTRY_ALIGN(aesni_gcm_encrypt, 32)
movq $192,%r10
vpshufb %xmm0,%xmm8,%xmm8
#ifdef HAVE_MOVBE
#if HAVE_SIMD(MOVBE)
#ifdef _KERNEL
testl $1,gcm_avx_can_use_movbe(%rip)
#else
@@ -1266,7 +1266,7 @@ SECTION_STATIC
#if defined(__linux__) && defined(HAVE_STACK_FRAME_NON_STANDARD) && \
! defined(HAVE_STACK_FRAME_NON_STANDARD_ASM)
.section .discard.func_stack_frame_non_standard, "aw"
#ifdef HAVE_MOVBE
#if HAVE_SIMD(MOVBE)
.long _aesni_ctr32_ghash_6x - .
#endif
.long _aesni_ctr32_ghash_no_movbe_6x - .
@@ -1279,4 +1279,4 @@ SECTION_STATIC
.section .note.GNU-stack,"",%progbits
#endif
#endif /* defined(__x86_64__) && defined(HAVE_AVX) && defined(HAVE_AES) ... */
#endif /* defined(__x86_64__) && HAVE_SIMD(AVX) && HAVE_SIMD(AES) ... */
module/icp/asm-x86_64/modes/gcm_pclmulqdq.S
+1 -1
View File
@@ -91,7 +91,7 @@ gcm_mul_pclmulqdq(uint64_t *x_in, uint64_t *y, uint64_t *res) {
(void) x_in, (void) y, (void) res;
}
#elif defined(HAVE_PCLMULQDQ) /* guard by instruction set */
#elif HAVE_SIMD(PCLMULQDQ) /* guard by instruction set */
#define _ASM
#include <sys/asm_linkage.h>
module/icp/asm-x86_64/modes/ghash-x86_64.S
+3 -3
View File
@@ -95,8 +95,8 @@
# and modified for ICP. Modification are kept at a bare minimum to ease later
# upstream merges.
#if defined(__x86_64__) && defined(HAVE_AVX) && \
defined(HAVE_AES) && defined(HAVE_PCLMULQDQ)
#if defined(__x86_64__) && HAVE_SIMD(AVX) && \
HAVE_SIMD(AES) && HAVE_SIMD(PCLMULQDQ)
#define _ASM
#include <sys/asm_linkage.h>
@@ -654,4 +654,4 @@ SET_OBJ(.Lrem_8bit)
.section .note.GNU-stack,"",%progbits
#endif
#endif /* defined(__x86_64__) && defined(HAVE_AVX) && defined(HAVE_AES) ... */
#endif /* defined(__x86_64__) && HAVE_SIMD(AVX) && HAVE_SIMD(AES) ... */
module/icp/asm-x86_64/sha2/sha512-x86_64.S
+3 -3
View File
@@ -4054,7 +4054,7 @@ ENTRY_ALIGN(zfs_sha512_transform_avx2, 64)
SET_SIZE(zfs_sha512_transform_avx2)
STACK_FRAME_NON_STANDARD zfs_sha512_transform_avx2
#ifdef HAVE_SHA512EXT
#if HAVE_SIMD(SHA512EXT)
ENTRY_ALIGN(zfs_sha512_transform_sha512ext, 64)
.cfi_startproc
ENDBR
@@ -4324,7 +4324,7 @@ ENTRY_ALIGN(zfs_sha512_transform_sha512ext, 64)
.cfi_endproc
SET_SIZE(zfs_sha512_transform_sha512ext)
STACK_FRAME_NON_STANDARD zfs_sha512_transform_sha512ext
#endif /* HAVE_SHA512EXT */
#endif /* HAVE_SIMD(SHA512EXT) */
/* Workaround for missing asm macro in RHEL 8. */
#if defined(__linux__) && defined(HAVE_STACK_FRAME_NON_STANDARD) && \
@@ -4333,7 +4333,7 @@ STACK_FRAME_NON_STANDARD zfs_sha512_transform_sha512ext
.long zfs_sha512_transform_x64 - .
.long zfs_sha512_transform_avx - .
.long zfs_sha512_transform_avx2 - .
#ifdef HAVE_SHA512EXT
#if HAVE_SIMD(SHA512EXT)
.long zfs_sha512_transform_sha512ext - .
#endif
#endif
module/icp/include/aes/aes_impl.h
+1 -1
View File
@@ -196,7 +196,7 @@ extern ASMABI void aes_encrypt_amd64(const uint32_t rk[], int Nr,
extern ASMABI void aes_decrypt_amd64(const uint32_t rk[], int Nr,
const uint32_t ct[4], uint32_t pt[4]);
#endif
#if defined(__x86_64) && defined(HAVE_AES)
#if defined(__x86_64) && HAVE_SIMD(AES)
extern const aes_impl_ops_t aes_aesni_impl;
#endif
module/icp/include/modes/gcm_impl.h
+1 -1
View File
@@ -55,7 +55,7 @@ typedef struct gcm_impl_ops {
} gcm_impl_ops_t;
extern const gcm_impl_ops_t gcm_generic_impl;
#if defined(__x86_64) && defined(HAVE_PCLMULQDQ)
#if defined(__x86_64) && HAVE_SIMD(PCLMULQDQ)
extern const gcm_impl_ops_t gcm_pclmulqdq_impl;
#endif
module/icp/include/modes/modes.h
+3 -3
View File
@@ -40,9 +40,9 @@ extern "C" {
* routines. AVX support should imply AES-NI and PCLMULQDQ, but make sure
* anyhow.
*/
#if defined(__x86_64__) && defined(HAVE_AVX) && \
defined(HAVE_AES) && defined(HAVE_PCLMULQDQ)
#define CAN_USE_GCM_ASM (HAVE_VAES && HAVE_VPCLMULQDQ ? 2 : 1)
#if defined(__x86_64__) && HAVE_SIMD(AVX) && \
HAVE_SIMD(AES) && HAVE_SIMD(PCLMULQDQ)
#define CAN_USE_GCM_ASM (HAVE_SIMD(VAES) && HAVE_SIMD(VPCLMULQDQ) ? 2 : 1)
extern boolean_t gcm_avx_can_use_movbe;
#endif