From 59493b63c18ea223857066218d6a58b67eb88159 Mon Sep 17 00:00:00 2001 From: Richard Yao Date: Mon, 5 Dec 2022 14:00:34 -0500 Subject: [PATCH] Micro-optimize fletcher4 calculations When processing abds, we execute 1 `kfpu_begin()`/`kfpu_end()` pair on every page in the abd. This is wasteful and slows down checksum performance versus what the benchmark claimed. We correct this by moving those calls to the init and fini functions. Also, we always check the buffer length against 0 before calling the non-scalar checksum functions. This means that we do not need to execute the loop condition for the first loop iteration. That allows us to micro-optimize the checksum calculations by switching to do-while loops. Note that we do not apply that micro-optimization to the scalar implementation because there is no check in `fletcher_4_incremental_native()`/`fletcher_4_incremental_byteswap()` against 0 sized buffers being passed. Reviewed-by: Alexander Motin Reviewed-by: Brian Behlendorf Signed-off-by: Richard Yao Closes #14247 --- module/zcommon/zfs_fletcher_aarch64_neon.c | 18 +++++---------- module/zcommon/zfs_fletcher_avx512.c | 26 +++++++--------------- module/zcommon/zfs_fletcher_intel.c | 18 +++++---------- module/zcommon/zfs_fletcher_sse.c | 26 +++++++--------------- module/zcommon/zfs_fletcher_superscalar.c | 8 +++---- module/zcommon/zfs_fletcher_superscalar4.c | 8 +++---- 6 files changed, 36 insertions(+), 68 deletions(-) diff --git a/module/zcommon/zfs_fletcher_aarch64_neon.c b/module/zcommon/zfs_fletcher_aarch64_neon.c index 3e14875d6..8f0339728 100644 --- a/module/zcommon/zfs_fletcher_aarch64_neon.c +++ b/module/zcommon/zfs_fletcher_aarch64_neon.c @@ -52,6 +52,7 @@ ZFS_NO_SANITIZE_UNDEFINED static void fletcher_4_aarch64_neon_init(fletcher_4_ctx_t *ctx) { + kfpu_begin(); memset(ctx->aarch64_neon, 0, 4 * sizeof (zfs_fletcher_aarch64_neon_t)); } @@ -69,6 +70,7 @@ fletcher_4_aarch64_neon_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp) 8 * ctx->aarch64_neon[3].v[1] - 8 * ctx->aarch64_neon[2].v[1] + ctx->aarch64_neon[1].v[1]; ZIO_SET_CHECKSUM(zcp, A, B, C, D); + kfpu_end(); } #define NEON_INIT_LOOP() \ @@ -146,17 +148,13 @@ unsigned char TMP2 __attribute__((vector_size(16))); unsigned char SRC __attribute__((vector_size(16))); #endif - kfpu_begin(); - NEON_INIT_LOOP(); - for (; ip < ipend; ip += 2) { + do { NEON_MAIN_LOOP(NEON_DONT_REVERSE); - } + } while ((ip += 2) < ipend); NEON_FINI_LOOP(); - - kfpu_end(); } static void @@ -185,17 +183,13 @@ unsigned char TMP2 __attribute__((vector_size(16))); unsigned char SRC __attribute__((vector_size(16))); #endif - kfpu_begin(); - NEON_INIT_LOOP(); - for (; ip < ipend; ip += 2) { + do { NEON_MAIN_LOOP(NEON_DO_REVERSE); - } + } while ((ip += 2) < ipend); NEON_FINI_LOOP(); - - kfpu_end(); } static boolean_t fletcher_4_aarch64_neon_valid(void) diff --git a/module/zcommon/zfs_fletcher_avx512.c b/module/zcommon/zfs_fletcher_avx512.c index 20c38d7e7..4a3d5cb24 100644 --- a/module/zcommon/zfs_fletcher_avx512.c +++ b/module/zcommon/zfs_fletcher_avx512.c @@ -39,6 +39,7 @@ ZFS_NO_SANITIZE_UNDEFINED static void fletcher_4_avx512f_init(fletcher_4_ctx_t *ctx) { + kfpu_begin(); memset(ctx->avx512, 0, 4 * sizeof (zfs_fletcher_avx512_t)); } @@ -72,6 +73,7 @@ fletcher_4_avx512f_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp) } ZIO_SET_CHECKSUM(zcp, A, B, C, D); + kfpu_end(); } #define FLETCHER_4_AVX512_RESTORE_CTX(ctx) \ @@ -96,21 +98,17 @@ fletcher_4_avx512f_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) const uint32_t *ip = buf; const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size); - kfpu_begin(); - FLETCHER_4_AVX512_RESTORE_CTX(ctx); - for (; ip < ipend; ip += 8) { + do { __asm("vpmovzxdq %0, %%zmm4"::"m" (*ip)); __asm("vpaddq %zmm4, %zmm0, %zmm0"); __asm("vpaddq %zmm0, %zmm1, %zmm1"); __asm("vpaddq %zmm1, %zmm2, %zmm2"); __asm("vpaddq %zmm2, %zmm3, %zmm3"); - } + } while ((ip += 8) < ipend); FLETCHER_4_AVX512_SAVE_CTX(ctx); - - kfpu_end(); } STACK_FRAME_NON_STANDARD(fletcher_4_avx512f_native); @@ -122,8 +120,6 @@ fletcher_4_avx512f_byteswap(fletcher_4_ctx_t *ctx, const void *buf, const uint32_t *ip = buf; const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size); - kfpu_begin(); - FLETCHER_4_AVX512_RESTORE_CTX(ctx); __asm("vpbroadcastq %0, %%zmm8" :: "r" (byteswap_mask)); @@ -131,7 +127,7 @@ fletcher_4_avx512f_byteswap(fletcher_4_ctx_t *ctx, const void *buf, __asm("vpsllq $16, %zmm8, %zmm10"); __asm("vpsllq $24, %zmm8, %zmm11"); - for (; ip < ipend; ip += 8) { + do { __asm("vpmovzxdq %0, %%zmm5"::"m" (*ip)); __asm("vpsrlq $24, %zmm5, %zmm6"); @@ -150,11 +146,9 @@ fletcher_4_avx512f_byteswap(fletcher_4_ctx_t *ctx, const void *buf, __asm("vpaddq %zmm0, %zmm1, %zmm1"); __asm("vpaddq %zmm1, %zmm2, %zmm2"); __asm("vpaddq %zmm2, %zmm3, %zmm3"); - } + } while ((ip += 8) < ipend); FLETCHER_4_AVX512_SAVE_CTX(ctx) - - kfpu_end(); } STACK_FRAME_NON_STANDARD(fletcher_4_avx512f_byteswap); @@ -189,13 +183,11 @@ fletcher_4_avx512bw_byteswap(fletcher_4_ctx_t *ctx, const void *buf, const uint32_t *ip = buf; const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size); - kfpu_begin(); - FLETCHER_4_AVX512_RESTORE_CTX(ctx); __asm("vmovdqu64 %0, %%zmm5" :: "m" (mask)); - for (; ip < ipend; ip += 8) { + do { __asm("vpmovzxdq %0, %%zmm4"::"m" (*ip)); __asm("vpshufb %zmm5, %zmm4, %zmm4"); @@ -204,11 +196,9 @@ fletcher_4_avx512bw_byteswap(fletcher_4_ctx_t *ctx, const void *buf, __asm("vpaddq %zmm0, %zmm1, %zmm1"); __asm("vpaddq %zmm1, %zmm2, %zmm2"); __asm("vpaddq %zmm2, %zmm3, %zmm3"); - } + } while ((ip += 8) < ipend); FLETCHER_4_AVX512_SAVE_CTX(ctx) - - kfpu_end(); } STACK_FRAME_NON_STANDARD(fletcher_4_avx512bw_byteswap); diff --git a/module/zcommon/zfs_fletcher_intel.c b/module/zcommon/zfs_fletcher_intel.c index 42b6309d3..c124d4928 100644 --- a/module/zcommon/zfs_fletcher_intel.c +++ b/module/zcommon/zfs_fletcher_intel.c @@ -51,6 +51,7 @@ ZFS_NO_SANITIZE_UNDEFINED static void fletcher_4_avx2_init(fletcher_4_ctx_t *ctx) { + kfpu_begin(); memset(ctx->avx, 0, 4 * sizeof (zfs_fletcher_avx_t)); } @@ -81,6 +82,7 @@ fletcher_4_avx2_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp) 64 * ctx->avx[3].v[3]; ZIO_SET_CHECKSUM(zcp, A, B, C, D); + kfpu_end(); } #define FLETCHER_4_AVX2_RESTORE_CTX(ctx) \ @@ -106,22 +108,18 @@ fletcher_4_avx2_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) const uint64_t *ip = buf; const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); - kfpu_begin(); - FLETCHER_4_AVX2_RESTORE_CTX(ctx); - for (; ip < ipend; ip += 2) { + do { asm volatile("vpmovzxdq %0, %%ymm4"::"m" (*ip)); asm volatile("vpaddq %ymm4, %ymm0, %ymm0"); asm volatile("vpaddq %ymm0, %ymm1, %ymm1"); asm volatile("vpaddq %ymm1, %ymm2, %ymm2"); asm volatile("vpaddq %ymm2, %ymm3, %ymm3"); - } + } while ((ip += 2) < ipend); FLETCHER_4_AVX2_SAVE_CTX(ctx); asm volatile("vzeroupper"); - - kfpu_end(); } static void @@ -134,13 +132,11 @@ fletcher_4_avx2_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) const uint64_t *ip = buf; const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); - kfpu_begin(); - FLETCHER_4_AVX2_RESTORE_CTX(ctx); asm volatile("vmovdqu %0, %%ymm5" :: "m" (mask)); - for (; ip < ipend; ip += 2) { + do { asm volatile("vpmovzxdq %0, %%ymm4"::"m" (*ip)); asm volatile("vpshufb %ymm5, %ymm4, %ymm4"); @@ -148,12 +144,10 @@ fletcher_4_avx2_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) asm volatile("vpaddq %ymm0, %ymm1, %ymm1"); asm volatile("vpaddq %ymm1, %ymm2, %ymm2"); asm volatile("vpaddq %ymm2, %ymm3, %ymm3"); - } + } while ((ip += 2) < ipend); FLETCHER_4_AVX2_SAVE_CTX(ctx); asm volatile("vzeroupper"); - - kfpu_end(); } static boolean_t fletcher_4_avx2_valid(void) diff --git a/module/zcommon/zfs_fletcher_sse.c b/module/zcommon/zfs_fletcher_sse.c index 791bbd49f..6c78830be 100644 --- a/module/zcommon/zfs_fletcher_sse.c +++ b/module/zcommon/zfs_fletcher_sse.c @@ -53,6 +53,7 @@ ZFS_NO_SANITIZE_UNDEFINED static void fletcher_4_sse2_init(fletcher_4_ctx_t *ctx) { + kfpu_begin(); memset(ctx->sse, 0, 4 * sizeof (zfs_fletcher_sse_t)); } @@ -80,6 +81,7 @@ fletcher_4_sse2_fini(fletcher_4_ctx_t *ctx, zio_cksum_t *zcp) 8 * ctx->sse[2].v[1] + ctx->sse[1].v[1]; ZIO_SET_CHECKSUM(zcp, A, B, C, D); + kfpu_end(); } #define FLETCHER_4_SSE_RESTORE_CTX(ctx) \ @@ -104,13 +106,11 @@ fletcher_4_sse2_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) const uint64_t *ip = buf; const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); - kfpu_begin(); - FLETCHER_4_SSE_RESTORE_CTX(ctx); asm volatile("pxor %xmm4, %xmm4"); - for (; ip < ipend; ip += 2) { + do { asm volatile("movdqu %0, %%xmm5" :: "m"(*ip)); asm volatile("movdqa %xmm5, %xmm6"); asm volatile("punpckldq %xmm4, %xmm5"); @@ -123,11 +123,9 @@ fletcher_4_sse2_native(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) asm volatile("paddq %xmm0, %xmm1"); asm volatile("paddq %xmm1, %xmm2"); asm volatile("paddq %xmm2, %xmm3"); - } + } while ((ip += 2) < ipend); FLETCHER_4_SSE_SAVE_CTX(ctx); - - kfpu_end(); } static void @@ -136,11 +134,9 @@ fletcher_4_sse2_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) const uint32_t *ip = buf; const uint32_t *ipend = (uint32_t *)((uint8_t *)ip + size); - kfpu_begin(); - FLETCHER_4_SSE_RESTORE_CTX(ctx); - for (; ip < ipend; ip += 2) { + do { uint32_t scratch1 = BSWAP_32(ip[0]); uint32_t scratch2 = BSWAP_32(ip[1]); asm volatile("movd %0, %%xmm5" :: "r"(scratch1)); @@ -150,11 +146,9 @@ fletcher_4_sse2_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) asm volatile("paddq %xmm0, %xmm1"); asm volatile("paddq %xmm1, %xmm2"); asm volatile("paddq %xmm2, %xmm3"); - } + } while ((ip += 2) < ipend); FLETCHER_4_SSE_SAVE_CTX(ctx); - - kfpu_end(); } static boolean_t fletcher_4_sse2_valid(void) @@ -186,14 +180,12 @@ fletcher_4_ssse3_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) const uint64_t *ip = buf; const uint64_t *ipend = (uint64_t *)((uint8_t *)ip + size); - kfpu_begin(); - FLETCHER_4_SSE_RESTORE_CTX(ctx); asm volatile("movdqu %0, %%xmm7"::"m" (mask)); asm volatile("pxor %xmm4, %xmm4"); - for (; ip < ipend; ip += 2) { + do { asm volatile("movdqu %0, %%xmm5"::"m" (*ip)); asm volatile("pshufb %xmm7, %xmm5"); asm volatile("movdqa %xmm5, %xmm6"); @@ -207,11 +199,9 @@ fletcher_4_ssse3_byteswap(fletcher_4_ctx_t *ctx, const void *buf, uint64_t size) asm volatile("paddq %xmm0, %xmm1"); asm volatile("paddq %xmm1, %xmm2"); asm volatile("paddq %xmm2, %xmm3"); - } + } while ((ip += 2) < ipend); FLETCHER_4_SSE_SAVE_CTX(ctx); - - kfpu_end(); } static boolean_t fletcher_4_ssse3_valid(void) diff --git a/module/zcommon/zfs_fletcher_superscalar.c b/module/zcommon/zfs_fletcher_superscalar.c index ba3fb54cb..67dc09592 100644 --- a/module/zcommon/zfs_fletcher_superscalar.c +++ b/module/zcommon/zfs_fletcher_superscalar.c @@ -89,7 +89,7 @@ fletcher_4_superscalar_native(fletcher_4_ctx_t *ctx, c2 = ctx->superscalar[2].v[1]; d2 = ctx->superscalar[3].v[1]; - for (; ip < ipend; ip += 2) { + do { a += ip[0]; a2 += ip[1]; b += a; @@ -98,7 +98,7 @@ fletcher_4_superscalar_native(fletcher_4_ctx_t *ctx, c2 += b2; d += c; d2 += c2; - } + } while ((ip += 2) < ipend); ctx->superscalar[0].v[0] = a; ctx->superscalar[1].v[0] = b; @@ -129,7 +129,7 @@ fletcher_4_superscalar_byteswap(fletcher_4_ctx_t *ctx, c2 = ctx->superscalar[2].v[1]; d2 = ctx->superscalar[3].v[1]; - for (; ip < ipend; ip += 2) { + do { a += BSWAP_32(ip[0]); a2 += BSWAP_32(ip[1]); b += a; @@ -138,7 +138,7 @@ fletcher_4_superscalar_byteswap(fletcher_4_ctx_t *ctx, c2 += b2; d += c; d2 += c2; - } + } while ((ip += 2) < ipend); ctx->superscalar[0].v[0] = a; ctx->superscalar[1].v[0] = b; diff --git a/module/zcommon/zfs_fletcher_superscalar4.c b/module/zcommon/zfs_fletcher_superscalar4.c index e3eda0295..d2067c12f 100644 --- a/module/zcommon/zfs_fletcher_superscalar4.c +++ b/module/zcommon/zfs_fletcher_superscalar4.c @@ -113,7 +113,7 @@ fletcher_4_superscalar4_native(fletcher_4_ctx_t *ctx, c4 = ctx->superscalar[2].v[3]; d4 = ctx->superscalar[3].v[3]; - for (; ip < ipend; ip += 4) { + do { a += ip[0]; a2 += ip[1]; a3 += ip[2]; @@ -130,7 +130,7 @@ fletcher_4_superscalar4_native(fletcher_4_ctx_t *ctx, d2 += c2; d3 += c3; d4 += c4; - } + } while ((ip += 4) < ipend); ctx->superscalar[0].v[0] = a; ctx->superscalar[1].v[0] = b; @@ -179,7 +179,7 @@ fletcher_4_superscalar4_byteswap(fletcher_4_ctx_t *ctx, c4 = ctx->superscalar[2].v[3]; d4 = ctx->superscalar[3].v[3]; - for (; ip < ipend; ip += 4) { + do { a += BSWAP_32(ip[0]); a2 += BSWAP_32(ip[1]); a3 += BSWAP_32(ip[2]); @@ -196,7 +196,7 @@ fletcher_4_superscalar4_byteswap(fletcher_4_ctx_t *ctx, d2 += c2; d3 += c3; d4 += c4; - } + } while ((ip += 4) < ipend); ctx->superscalar[0].v[0] = a; ctx->superscalar[1].v[0] = b;