Permalink
Browse files

bn/asm/rsaz-avx2.pl: fix digit correction bug in rsaz_1024_mul_avx2.

Credit to OSS-Fuzz for finding this.

CVE-2017-3738

Reviewed-by: Rich Salz <rsalz@openssl.org>
  • Loading branch information...
dot-asm authored and mattcaswell committed Nov 24, 2017
1 parent 7ed680c commit ca51bafc1a88d8b8348f5fd97adc5d6ca93f8e76
Showing with 7 additions and 8 deletions.
  1. +7 −8 crypto/bn/asm/rsaz-avx2.pl
@@ -239,7 +239,7 @@
vmovdqu 32*8-128($ap), $ACC8
lea 192(%rsp), $tp0 # 64+128=192
vpbroadcastq .Land_mask(%rip), $AND_MASK
vmovdqu .Land_mask(%rip), $AND_MASK
jmp .LOOP_GRANDE_SQR_1024
.align 32
@@ -1070,10 +1070,10 @@
vpmuludq 32*6-128($np),$Yi,$TEMP1
vpaddq $TEMP1,$ACC6,$ACC6
vpmuludq 32*7-128($np),$Yi,$TEMP2
vpblendd \$3, $ZERO, $ACC9, $ACC9 # correct $ACC3
vpblendd \$3, $ZERO, $ACC9, $TEMP1 # correct $ACC3
vpaddq $TEMP2,$ACC7,$ACC7
vpmuludq 32*8-128($np),$Yi,$TEMP0
vpaddq $ACC9, $ACC3, $ACC3 # correct $ACC3
vpaddq $TEMP1, $ACC3, $ACC3 # correct $ACC3
vpaddq $TEMP0,$ACC8,$ACC8
mov %rbx, %rax
@@ -1086,7 +1086,9 @@
vmovdqu -8+32*2-128($ap),$TEMP2
mov $r1, %rax
vpblendd \$0xfc, $ZERO, $ACC9, $ACC9 # correct $ACC3
imull $n0, %eax
vpaddq $ACC9,$ACC4,$ACC4 # correct $ACC3
and \$0x1fffffff, %eax
imulq 16-128($ap),%rbx
@@ -1322,15 +1324,12 @@
# But as we underutilize resources, it's possible to correct in
# each iteration with marginal performance loss. But then, as
# we do it in each iteration, we can correct less digits, and
# avoid performance penalties completely. Also note that we
# correct only three digits out of four. This works because
# most significant digit is subjected to less additions.
# avoid performance penalties completely.
$TEMP0 = $ACC9;
$TEMP3 = $Bi;
$TEMP4 = $Yi;
$code.=<<___;
vpermq \$0, $AND_MASK, $AND_MASK
vpaddq (%rsp), $TEMP1, $ACC0
vpsrlq \$29, $ACC0, $TEMP1
@@ -1763,7 +1762,7 @@
.align 64
.Land_mask:
.quad 0x1fffffff,0x1fffffff,0x1fffffff,-1
.quad 0x1fffffff,0x1fffffff,0x1fffffff,0x1fffffff
.Lscatter_permd:
.long 0,2,4,6,7,7,7,7
.Lgather_permd:

0 comments on commit ca51baf

Please sign in to comment.