|
 |
94eca17 |
commit df5de87260dba479873b2850bbe5c0b81c2376f6
|
|
|
94eca17 |
Author: Noah Goldstein <goldstein.w.n@gmail.com>
|
|
|
94eca17 |
Date: Fri Apr 15 12:28:01 2022 -0500
|
|
|
94eca17 |
|
|
|
94eca17 |
x86: Cleanup page cross code in memcmp-avx2-movbe.S
|
|
|
94eca17 |
|
|
|
94eca17 |
Old code was both inefficient and wasted code size. New code (-62
|
|
|
94eca17 |
bytes) and comparable or better performance in the page cross case.
|
|
|
94eca17 |
|
|
|
94eca17 |
geometric_mean(N=20) of page cross cases New / Original: 0.960
|
|
|
94eca17 |
|
|
|
94eca17 |
size, align0, align1, ret, New Time/Old Time
|
|
|
94eca17 |
1, 4095, 0, 0, 1.001
|
|
|
94eca17 |
1, 4095, 0, 1, 0.999
|
|
|
94eca17 |
1, 4095, 0, -1, 1.0
|
|
|
94eca17 |
2, 4094, 0, 0, 1.0
|
|
|
94eca17 |
2, 4094, 0, 1, 1.0
|
|
|
94eca17 |
2, 4094, 0, -1, 1.0
|
|
|
94eca17 |
3, 4093, 0, 0, 1.0
|
|
|
94eca17 |
3, 4093, 0, 1, 1.0
|
|
|
94eca17 |
3, 4093, 0, -1, 1.0
|
|
|
94eca17 |
4, 4092, 0, 0, 0.987
|
|
|
94eca17 |
4, 4092, 0, 1, 1.0
|
|
|
94eca17 |
4, 4092, 0, -1, 1.0
|
|
|
94eca17 |
5, 4091, 0, 0, 0.984
|
|
|
94eca17 |
5, 4091, 0, 1, 1.002
|
|
|
94eca17 |
5, 4091, 0, -1, 1.005
|
|
|
94eca17 |
6, 4090, 0, 0, 0.993
|
|
|
94eca17 |
6, 4090, 0, 1, 1.001
|
|
|
94eca17 |
6, 4090, 0, -1, 1.003
|
|
|
94eca17 |
7, 4089, 0, 0, 0.991
|
|
|
94eca17 |
7, 4089, 0, 1, 1.0
|
|
|
94eca17 |
7, 4089, 0, -1, 1.001
|
|
|
94eca17 |
8, 4088, 0, 0, 0.875
|
|
|
94eca17 |
8, 4088, 0, 1, 0.881
|
|
|
94eca17 |
8, 4088, 0, -1, 0.888
|
|
|
94eca17 |
9, 4087, 0, 0, 0.872
|
|
|
94eca17 |
9, 4087, 0, 1, 0.879
|
|
|
94eca17 |
9, 4087, 0, -1, 0.883
|
|
|
94eca17 |
10, 4086, 0, 0, 0.878
|
|
|
94eca17 |
10, 4086, 0, 1, 0.886
|
|
|
94eca17 |
10, 4086, 0, -1, 0.873
|
|
|
94eca17 |
11, 4085, 0, 0, 0.878
|
|
|
94eca17 |
11, 4085, 0, 1, 0.881
|
|
|
94eca17 |
11, 4085, 0, -1, 0.879
|
|
|
94eca17 |
12, 4084, 0, 0, 0.873
|
|
|
94eca17 |
12, 4084, 0, 1, 0.889
|
|
|
94eca17 |
12, 4084, 0, -1, 0.875
|
|
|
94eca17 |
13, 4083, 0, 0, 0.873
|
|
|
94eca17 |
13, 4083, 0, 1, 0.863
|
|
|
94eca17 |
13, 4083, 0, -1, 0.863
|
|
|
94eca17 |
14, 4082, 0, 0, 0.838
|
|
|
94eca17 |
14, 4082, 0, 1, 0.869
|
|
|
94eca17 |
14, 4082, 0, -1, 0.877
|
|
|
94eca17 |
15, 4081, 0, 0, 0.841
|
|
|
94eca17 |
15, 4081, 0, 1, 0.869
|
|
|
94eca17 |
15, 4081, 0, -1, 0.876
|
|
|
94eca17 |
16, 4080, 0, 0, 0.988
|
|
|
94eca17 |
16, 4080, 0, 1, 0.99
|
|
|
94eca17 |
16, 4080, 0, -1, 0.989
|
|
|
94eca17 |
17, 4079, 0, 0, 0.978
|
|
|
94eca17 |
17, 4079, 0, 1, 0.981
|
|
|
94eca17 |
17, 4079, 0, -1, 0.98
|
|
|
94eca17 |
18, 4078, 0, 0, 0.981
|
|
|
94eca17 |
18, 4078, 0, 1, 0.98
|
|
|
94eca17 |
18, 4078, 0, -1, 0.985
|
|
|
94eca17 |
19, 4077, 0, 0, 0.977
|
|
|
94eca17 |
19, 4077, 0, 1, 0.979
|
|
|
94eca17 |
19, 4077, 0, -1, 0.986
|
|
|
94eca17 |
20, 4076, 0, 0, 0.977
|
|
|
94eca17 |
20, 4076, 0, 1, 0.986
|
|
|
94eca17 |
20, 4076, 0, -1, 0.984
|
|
|
94eca17 |
21, 4075, 0, 0, 0.977
|
|
|
94eca17 |
21, 4075, 0, 1, 0.983
|
|
|
94eca17 |
21, 4075, 0, -1, 0.988
|
|
|
94eca17 |
22, 4074, 0, 0, 0.983
|
|
|
94eca17 |
22, 4074, 0, 1, 0.994
|
|
|
94eca17 |
22, 4074, 0, -1, 0.993
|
|
|
94eca17 |
23, 4073, 0, 0, 0.98
|
|
|
94eca17 |
23, 4073, 0, 1, 0.992
|
|
|
94eca17 |
23, 4073, 0, -1, 0.995
|
|
|
94eca17 |
24, 4072, 0, 0, 0.989
|
|
|
94eca17 |
24, 4072, 0, 1, 0.989
|
|
|
94eca17 |
24, 4072, 0, -1, 0.991
|
|
|
94eca17 |
25, 4071, 0, 0, 0.99
|
|
|
94eca17 |
25, 4071, 0, 1, 0.999
|
|
|
94eca17 |
25, 4071, 0, -1, 0.996
|
|
|
94eca17 |
26, 4070, 0, 0, 0.993
|
|
|
94eca17 |
26, 4070, 0, 1, 0.995
|
|
|
94eca17 |
26, 4070, 0, -1, 0.998
|
|
|
94eca17 |
27, 4069, 0, 0, 0.993
|
|
|
94eca17 |
27, 4069, 0, 1, 0.999
|
|
|
94eca17 |
27, 4069, 0, -1, 1.0
|
|
|
94eca17 |
28, 4068, 0, 0, 0.997
|
|
|
94eca17 |
28, 4068, 0, 1, 1.0
|
|
|
94eca17 |
28, 4068, 0, -1, 0.999
|
|
|
94eca17 |
29, 4067, 0, 0, 0.996
|
|
|
94eca17 |
29, 4067, 0, 1, 0.999
|
|
|
94eca17 |
29, 4067, 0, -1, 0.999
|
|
|
94eca17 |
30, 4066, 0, 0, 0.991
|
|
|
94eca17 |
30, 4066, 0, 1, 1.001
|
|
|
94eca17 |
30, 4066, 0, -1, 0.999
|
|
|
94eca17 |
31, 4065, 0, 0, 0.988
|
|
|
94eca17 |
31, 4065, 0, 1, 0.998
|
|
|
94eca17 |
31, 4065, 0, -1, 0.998
|
|
|
94eca17 |
Reviewed-by: H.J. Lu <hjl.tools@gmail.com>
|
|
|
94eca17 |
|
|
|
94eca17 |
(cherry picked from commit 23102686ec67b856a2d4fd25ddaa1c0b8d175c4f)
|
|
|
94eca17 |
|
|
|
94eca17 |
diff --git a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S
|
|
|
94eca17 |
index 2621ec907aedb781..ec9cf0852edf216d 100644
|
|
|
94eca17 |
--- a/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S
|
|
|
94eca17 |
+++ b/sysdeps/x86_64/multiarch/memcmp-avx2-movbe.S
|
|
|
94eca17 |
@@ -429,22 +429,21 @@ L(page_cross_less_vec):
|
|
|
94eca17 |
# ifndef USE_AS_WMEMCMP
|
|
|
94eca17 |
cmpl $8, %edx
|
|
|
94eca17 |
jae L(between_8_15)
|
|
|
94eca17 |
+ /* Fall through for [4, 7]. */
|
|
|
94eca17 |
cmpl $4, %edx
|
|
|
94eca17 |
- jae L(between_4_7)
|
|
|
94eca17 |
+ jb L(between_2_3)
|
|
|
94eca17 |
|
|
|
94eca17 |
- /* Load as big endian to avoid branches. */
|
|
|
94eca17 |
- movzwl (%rdi), %eax
|
|
|
94eca17 |
- movzwl (%rsi), %ecx
|
|
|
94eca17 |
- shll $8, %eax
|
|
|
94eca17 |
- shll $8, %ecx
|
|
|
94eca17 |
- bswap %eax
|
|
|
94eca17 |
- bswap %ecx
|
|
|
94eca17 |
- movzbl -1(%rdi, %rdx), %edi
|
|
|
94eca17 |
- movzbl -1(%rsi, %rdx), %esi
|
|
|
94eca17 |
- orl %edi, %eax
|
|
|
94eca17 |
- orl %esi, %ecx
|
|
|
94eca17 |
- /* Subtraction is okay because the upper 8 bits are zero. */
|
|
|
94eca17 |
- subl %ecx, %eax
|
|
|
94eca17 |
+ movbe (%rdi), %eax
|
|
|
94eca17 |
+ movbe (%rsi), %ecx
|
|
|
94eca17 |
+ shlq $32, %rax
|
|
|
94eca17 |
+ shlq $32, %rcx
|
|
|
94eca17 |
+ movbe -4(%rdi, %rdx), %edi
|
|
|
94eca17 |
+ movbe -4(%rsi, %rdx), %esi
|
|
|
94eca17 |
+ orq %rdi, %rax
|
|
|
94eca17 |
+ orq %rsi, %rcx
|
|
|
94eca17 |
+ subq %rcx, %rax
|
|
|
94eca17 |
+ /* Fast path for return zero. */
|
|
|
94eca17 |
+ jnz L(ret_nonzero)
|
|
|
94eca17 |
/* No ymm register was touched. */
|
|
|
94eca17 |
ret
|
|
|
94eca17 |
|
|
|
94eca17 |
@@ -457,9 +456,33 @@ L(one_or_less):
|
|
|
94eca17 |
/* No ymm register was touched. */
|
|
|
94eca17 |
ret
|
|
|
94eca17 |
|
|
|
94eca17 |
+ .p2align 4,, 5
|
|
|
94eca17 |
+L(ret_nonzero):
|
|
|
94eca17 |
+ sbbl %eax, %eax
|
|
|
94eca17 |
+ orl $1, %eax
|
|
|
94eca17 |
+ /* No ymm register was touched. */
|
|
|
94eca17 |
+ ret
|
|
|
94eca17 |
+
|
|
|
94eca17 |
+ .p2align 4,, 2
|
|
|
94eca17 |
+L(zero):
|
|
|
94eca17 |
+ xorl %eax, %eax
|
|
|
94eca17 |
+ /* No ymm register was touched. */
|
|
|
94eca17 |
+ ret
|
|
|
94eca17 |
+
|
|
|
94eca17 |
.p2align 4
|
|
|
94eca17 |
L(between_8_15):
|
|
|
94eca17 |
-# endif
|
|
|
94eca17 |
+ movbe (%rdi), %rax
|
|
|
94eca17 |
+ movbe (%rsi), %rcx
|
|
|
94eca17 |
+ subq %rcx, %rax
|
|
|
94eca17 |
+ jnz L(ret_nonzero)
|
|
|
94eca17 |
+ movbe -8(%rdi, %rdx), %rax
|
|
|
94eca17 |
+ movbe -8(%rsi, %rdx), %rcx
|
|
|
94eca17 |
+ subq %rcx, %rax
|
|
|
94eca17 |
+ /* Fast path for return zero. */
|
|
|
94eca17 |
+ jnz L(ret_nonzero)
|
|
|
94eca17 |
+ /* No ymm register was touched. */
|
|
|
94eca17 |
+ ret
|
|
|
94eca17 |
+# else
|
|
|
94eca17 |
/* If USE_AS_WMEMCMP fall through into 8-15 byte case. */
|
|
|
94eca17 |
vmovq (%rdi), %xmm1
|
|
|
94eca17 |
vmovq (%rsi), %xmm2
|
|
|
94eca17 |
@@ -475,16 +498,13 @@ L(between_8_15):
|
|
|
94eca17 |
VPCMPEQ %xmm1, %xmm2, %xmm2
|
|
|
94eca17 |
vpmovmskb %xmm2, %eax
|
|
|
94eca17 |
subl $0xffff, %eax
|
|
|
94eca17 |
+ /* Fast path for return zero. */
|
|
|
94eca17 |
jnz L(return_vec_0)
|
|
|
94eca17 |
/* No ymm register was touched. */
|
|
|
94eca17 |
ret
|
|
|
94eca17 |
+# endif
|
|
|
94eca17 |
|
|
|
94eca17 |
- .p2align 4
|
|
|
94eca17 |
-L(zero):
|
|
|
94eca17 |
- xorl %eax, %eax
|
|
|
94eca17 |
- ret
|
|
|
94eca17 |
-
|
|
|
94eca17 |
- .p2align 4
|
|
|
94eca17 |
+ .p2align 4,, 10
|
|
|
94eca17 |
L(between_16_31):
|
|
|
94eca17 |
/* From 16 to 31 bytes. No branch when size == 16. */
|
|
|
94eca17 |
vmovdqu (%rsi), %xmm2
|
|
|
94eca17 |
@@ -501,11 +521,17 @@ L(between_16_31):
|
|
|
94eca17 |
VPCMPEQ (%rdi), %xmm2, %xmm2
|
|
|
94eca17 |
vpmovmskb %xmm2, %eax
|
|
|
94eca17 |
subl $0xffff, %eax
|
|
|
94eca17 |
+ /* Fast path for return zero. */
|
|
|
94eca17 |
jnz L(return_vec_0)
|
|
|
94eca17 |
/* No ymm register was touched. */
|
|
|
94eca17 |
ret
|
|
|
94eca17 |
|
|
|
94eca17 |
# ifdef USE_AS_WMEMCMP
|
|
|
94eca17 |
+ .p2align 4,, 2
|
|
|
94eca17 |
+L(zero):
|
|
|
94eca17 |
+ xorl %eax, %eax
|
|
|
94eca17 |
+ ret
|
|
|
94eca17 |
+
|
|
|
94eca17 |
.p2align 4
|
|
|
94eca17 |
L(one_or_less):
|
|
|
94eca17 |
jb L(zero)
|
|
|
94eca17 |
@@ -520,22 +546,20 @@ L(one_or_less):
|
|
|
94eca17 |
# else
|
|
|
94eca17 |
|
|
|
94eca17 |
.p2align 4
|
|
|
94eca17 |
-L(between_4_7):
|
|
|
94eca17 |
- /* Load as big endian with overlapping movbe to avoid branches.
|
|
|
94eca17 |
- */
|
|
|
94eca17 |
- movbe (%rdi), %eax
|
|
|
94eca17 |
- movbe (%rsi), %ecx
|
|
|
94eca17 |
- shlq $32, %rax
|
|
|
94eca17 |
- shlq $32, %rcx
|
|
|
94eca17 |
- movbe -4(%rdi, %rdx), %edi
|
|
|
94eca17 |
- movbe -4(%rsi, %rdx), %esi
|
|
|
94eca17 |
- orq %rdi, %rax
|
|
|
94eca17 |
- orq %rsi, %rcx
|
|
|
94eca17 |
- subq %rcx, %rax
|
|
|
94eca17 |
- jz L(zero_4_7)
|
|
|
94eca17 |
- sbbl %eax, %eax
|
|
|
94eca17 |
- orl $1, %eax
|
|
|
94eca17 |
-L(zero_4_7):
|
|
|
94eca17 |
+L(between_2_3):
|
|
|
94eca17 |
+ /* Load as big endian to avoid branches. */
|
|
|
94eca17 |
+ movzwl (%rdi), %eax
|
|
|
94eca17 |
+ movzwl (%rsi), %ecx
|
|
|
94eca17 |
+ bswap %eax
|
|
|
94eca17 |
+ bswap %ecx
|
|
|
94eca17 |
+ shrl %eax
|
|
|
94eca17 |
+ shrl %ecx
|
|
|
94eca17 |
+ movzbl -1(%rdi, %rdx), %edi
|
|
|
94eca17 |
+ movzbl -1(%rsi, %rdx), %esi
|
|
|
94eca17 |
+ orl %edi, %eax
|
|
|
94eca17 |
+ orl %esi, %ecx
|
|
|
94eca17 |
+ /* Subtraction is okay because the upper bit is zero. */
|
|
|
94eca17 |
+ subl %ecx, %eax
|
|
|
94eca17 |
/* No ymm register was touched. */
|
|
|
94eca17 |
ret
|
|
|
94eca17 |
# endif
|