.text
.globl ossl_rsaz_avxifma_eligible
.type ossl_rsaz_avxifma_eligible,@function
.align 32
ossl_rsaz_avxifma_eligible:
movl OPENSSL_ia32cap_P+20(%rip),%ecx
xorl %eax,%eax
andl $8388608,%ecx
cmpl $8388608,%ecx
cmovel %ecx,%eax
.byte 0xf3,0xc3
.size ossl_rsaz_avxifma_eligible, .-ossl_rsaz_avxifma_eligible
.text
.globl ossl_rsaz_amm52x20_x1_avxifma256
.type ossl_rsaz_amm52x20_x1_avxifma256,@function
.align 32
ossl_rsaz_amm52x20_x1_avxifma256:
.cfi_startproc
.byte 243,15,30,250
pushq %rbx
.cfi_adjust_cfa_offset 8
.cfi_offset %rbx,-16
pushq %rbp
.cfi_adjust_cfa_offset 8
.cfi_offset %rbp,-24
pushq %r12
.cfi_adjust_cfa_offset 8
.cfi_offset %r12,-32
pushq %r13
.cfi_adjust_cfa_offset 8
.cfi_offset %r13,-40
pushq %r14
.cfi_adjust_cfa_offset 8
.cfi_offset %r14,-48
pushq %r15
.cfi_adjust_cfa_offset 8
.cfi_offset %r15,-56
.Lossl_rsaz_amm52x20_x1_avxifma256_body:
vpxor %ymm0,%ymm0,%ymm0
vmovapd %ymm0,%ymm3
vmovapd %ymm0,%ymm5
vmovapd %ymm0,%ymm6
vmovapd %ymm0,%ymm7
vmovapd %ymm0,%ymm8
xorl %r9d,%r9d
movq %rdx,%r11
movq $0xfffffffffffff,%rax
movl $5,%ebx
.align 32
.Lloop5:
movq 0(%r11),%r13
vpbroadcastq 0(%r11),%ymm1
movq 0(%rsi),%rdx
mulxq %r13,%r13,%r12
addq %r13,%r9
movq %r12,%r10
adcq $0,%r10
movq %r8,%r13
imulq %r9,%r13
andq %rax,%r13
vmovq %r13,%xmm2
vpbroadcastq %xmm2,%ymm2
movq 0(%rcx),%rdx
mulxq %r13,%r13,%r12
addq %r13,%r9
adcq %r12,%r10
shrq $52,%r9
salq $12,%r10
orq %r10,%r9
leaq -168(%rsp),%rsp
{vex} vpmadd52luq 0(%rsi),%ymm1,%ymm3
{vex} vpmadd52luq 32(%rsi),%ymm1,%ymm5
{vex} vpmadd52luq 64(%rsi),%ymm1,%ymm6
{vex} vpmadd52luq 96(%rsi),%ymm1,%ymm7
{vex} vpmadd52luq 128(%rsi),%ymm1,%ymm8
{vex} vpmadd52luq 0(%rcx),%ymm2,%ymm3
{vex} vpmadd52luq 32(%rcx),%ymm2,%ymm5
{vex} vpmadd52luq 64(%rcx),%ymm2,%ymm6
{vex} vpmadd52luq 96(%rcx),%ymm2,%ymm7
{vex} vpmadd52luq 128(%rcx),%ymm2,%ymm8
vmovdqu %ymm3,0(%rsp)
vmovdqu %ymm5,32(%rsp)
vmovdqu %ymm6,64(%rsp)
vmovdqu %ymm7,96(%rsp)
vmovdqu %ymm8,128(%rsp)
movq $0,160(%rsp)
vmovdqu 8(%rsp),%ymm3
vmovdqu 40(%rsp),%ymm5
vmovdqu 72(%rsp),%ymm6
vmovdqu 104(%rsp),%ymm7
vmovdqu 136(%rsp),%ymm8
addq 8(%rsp),%r9
{vex} vpmadd52huq 0(%rsi),%ymm1,%ymm3
{vex} vpmadd52huq 32(%rsi),%ymm1,%ymm5
{vex} vpmadd52huq 64(%rsi),%ymm1,%ymm6
{vex} vpmadd52huq 96(%rsi),%ymm1,%ymm7
{vex} vpmadd52huq 128(%rsi),%ymm1,%ymm8
{vex} vpmadd52huq 0(%rcx),%ymm2,%ymm3
{vex} vpmadd52huq 32(%rcx),%ymm2,%ymm5
{vex} vpmadd52huq 64(%rcx),%ymm2,%ymm6
{vex} vpmadd52huq 96(%rcx),%ymm2,%ymm7
{vex} vpmadd52huq 128(%rcx),%ymm2,%ymm8
leaq 168(%rsp),%rsp
movq 8(%r11),%r13
vpbroadcastq 8(%r11),%ymm1
movq 0(%rsi),%rdx
mulxq %r13,%r13,%r12
addq %r13,%r9
movq %r12,%r10
adcq $0,%r10
movq %r8,%r13
imulq %r9,%r13
andq %rax,%r13
vmovq %r13,%xmm2
vpbroadcastq %xmm2,%ymm2
movq 0(%rcx),%rdx
mulxq %r13,%r13,%r12
addq %r13,%r9
adcq %r12,%r10
shrq $52,%r9
salq $12,%r10
orq %r10,%r9
leaq -168(%rsp),%rsp
{vex} vpmadd52luq 0(%rsi),%ymm1,%ymm3
{vex} vpmadd52luq 32(%rsi),%ymm1,%ymm5
{vex} vpmadd52luq 64(%rsi),%ymm1,%ymm6
{vex} vpmadd52luq 96(%rsi),%ymm1,%ymm7
{vex} vpmadd52luq 128(%rsi),%ymm1,%ymm8
{vex} vpmadd52luq 0(%rcx),%ymm2,%ymm3
{vex} vpmadd52luq 32(%rcx),%ymm2,%ymm5
{vex} vpmadd52luq 64(%rcx),%ymm2,%ymm6
{vex} vpmadd52luq 96(%rcx),%ymm2,%ymm7
{vex} vpmadd52luq 128(%rcx),%ymm2,%ymm8
vmovdqu %ymm3,0(%rsp)
vmovdqu %ymm5,32(%rsp)
vmovdqu %ymm6,64(%rsp)
vmovdqu %ymm7,96(%rsp)
vmovdqu %ymm8,128(%rsp)
movq $0,160(%rsp)
vmovdqu 8(%rsp),%ymm3
vmovdqu 40(%rsp),%ymm5
vmovdqu 72(%rsp),%ymm6
vmovdqu 104(%rsp),%ymm7
vmovdqu 136(%rsp),%ymm8
addq 8(%rsp),%r9
{vex} vpmadd52huq 0(%rsi),%ymm1,%ymm3
{vex} vpmadd52huq 32(%rsi),%ymm1,%ymm5
{vex} vpmadd52huq 64(%rsi),%ymm1,%ymm6
{vex} vpmadd52huq 96(%rsi),%ymm1,%ymm7
{vex} vpmadd52huq 128(%rsi),%ymm1,%ymm8
{vex} vpmadd52huq 0(%rcx),%ymm2,%ymm3
{vex} vpmadd52huq 32(%rcx),%ymm2,%ymm5
{vex} vpmadd52huq 64(%rcx),%ymm2,%ymm6
{vex} vpmadd52huq 96(%rcx),%ymm2,%ymm7
{vex} vpmadd52huq 128(%rcx),%ymm2,%ymm8
leaq 168(%rsp),%rsp
movq 16(%r11),%r13
vpbroadcastq 16(%r11),%ymm1
movq 0(%rsi),%rdx
mulxq %r13,%r13,%r12
addq %r13,%r9
movq %r12,%r10
adcq $0,%r10
movq %r8,%r13
imulq %r9,%r13
andq %rax,%r13
vmovq %r13,%xmm2
vpbroadcastq %xmm2,%ymm2
movq 0(%rcx),%rdx
mulxq %r13,%r13,%r12
addq %r13,%r9
adcq %r12,%r10
shrq $52,%r9
salq $12,%r10
orq %r10,%r9
leaq -168(%rsp),%rsp
{vex} vpmadd52luq 0(%rsi),%ymm1,%ymm3
{vex} vpmadd52luq 32(%rsi),%ymm1,%ymm5
{vex} vpmadd52luq 64(%rsi),%ymm1,%ymm6
{vex} vpmadd52luq 96(%rsi),%ymm1,%ymm7
{vex} vpmadd52luq 128(%rsi),%ymm1,%ymm8
{vex} vpmadd52luq 0(%rcx),%ymm2,%ymm3
{vex} vpmadd52luq 32(%rcx),%ymm2,%ymm5
{vex} vpmadd52luq 64(%rcx),%ymm2,%ymm6
{vex} vpmadd52luq 96(%rcx),%ymm2,%ymm7
{vex} vpmadd52luq 128(%rcx),%ymm2,%ymm8
vmovdqu %ymm3,0(%rsp)
vmovdqu %ymm5,32(%rsp)
vmovdqu %ymm6,64(%rsp)
vmovdqu %ymm7,96(%rsp)
vmovdqu %ymm8,128(%rsp)
movq $0,160(%rsp)
vmovdqu 8(%rsp),%ymm3
vmovdqu 40(%rsp),%ymm5
vmovdqu 72(%rsp),%ymm6
vmovdqu 104(%rsp),%ymm7
vmovdqu 136(%rsp),%ymm8
addq 8(%rsp),%r9
{vex} vpmadd52huq 0(%rsi),%ymm1,%ymm3
{vex} vpmadd52huq 32(%rsi),%ymm1,%ymm5
{vex} vpmadd52huq 64(%rsi),%ymm1,%ymm6
{vex} vpmadd52huq 96(%rsi),%ymm1,%ymm7
{vex} vpmadd52huq 128(%rsi),%ymm1,%ymm8
{vex} vpmadd52huq 0(%rcx),%ymm2,%ymm3
{vex} vpmadd52huq 32(%rcx),%ymm2,%ymm5
{vex} vpmadd52huq 64(%rcx),%ymm2,%ymm6
{vex} vpmadd52huq 96(%rcx),%ymm2,%ymm7
{vex} vpmadd52huq 128(%rcx),%ymm2,%ymm8
leaq 168(%rsp),%rsp
movq 24(%r11),%r13
vpbroadcastq 24(%r11),%ymm1
movq 0(%rsi),%rdx
mulxq %r13,%r13,%r12
addq %r13,%r9
movq %r12,%r10
adcq $0,%r10
movq %r8,%r13
imulq %r9,%r13
andq %rax,%r13
vmovq %r13,%xmm2
vpbroadcastq %xmm2,%ymm2
movq 0(%rcx),%rdx
mulxq %r13,%r13,%r12
addq %r13,%r9
adcq %r12,%r10
shrq $52,%r9
salq $12,%r10
orq %r10,%r9
leaq -168(%rsp),%rsp
{vex} vpmadd52luq 0(%rsi),%ymm1,%ymm3
{vex} vpmadd52luq 32(%rsi),%ymm1,%ymm5
{vex} vpmadd52luq 64(%rsi),%ymm1,%ymm6
{vex} vpmadd52luq 96(%rsi),%ymm1,%ymm7
{vex} vpmadd52luq 128(%rsi),%ymm1,%ymm8
{vex} vpmadd52luq 0(%rcx),%ymm2,%ymm3
{vex} vpmadd52luq 32(%rcx),%ymm2,%ymm5
{vex} vpmadd52luq 64(%rcx),%ymm2,%ymm6
{vex} vpmadd52luq 96(%rcx),%ymm2,%ymm7
{vex} vpmadd52luq 128(%rcx),%ymm2,%ymm8
vmovdqu %ymm3,0(%rsp)
vmovdqu %ymm5,32(%rsp)
vmovdqu %ymm6,64(%rsp)
vmovdqu %ymm7,96(%rsp)
vmovdqu %ymm8,128(%rsp)
movq $0,160(%rsp)
vmovdqu 8(%rsp),%ymm3
vmovdqu 40(%rsp),%ymm5
vmovdqu 72(%rsp),%ymm6
vmovdqu 104(%rsp),%ymm7
vmovdqu 136(%rsp),%ymm8
addq 8(%rsp),%r9
{vex} vpmadd52huq 0(%rsi),%ymm1,%ymm3
{vex} vpmadd52huq 32(%rsi),%ymm1,%ymm5
{vex} vpmadd52huq 64(%rsi),%ymm1,%ymm6
{vex} vpmadd52huq 96(%rsi),%ymm1,%ymm7
{vex} vpmadd52huq 128(%rsi),%ymm1,%ymm8
{vex} vpmadd52huq 0(%rcx),%ymm2,%ymm3
{vex} vpmadd52huq 32(%rcx),%ymm2,%ymm5
{vex} vpmadd52huq 64(%rcx),%ymm2,%ymm6
{vex} vpmadd52huq 96(%rcx),%ymm2,%ymm7
{vex} vpmadd52huq 128(%rcx),%ymm2,%ymm8
leaq 168(%rsp),%rsp
leaq 32(%r11),%r11
decl %ebx
jne .Lloop5
vmovq %r9,%xmm0
vpbroadcastq %xmm0,%ymm0
vpblendd $3,%ymm0,%ymm3,%ymm3
vpsrlq $52,%ymm3,%ymm0
vpsrlq $52,%ymm5,%ymm1
vpsrlq $52,%ymm6,%ymm2
vpsrlq $52,%ymm7,%ymm13
vpsrlq $52,%ymm8,%ymm14
vpermq $144,%ymm14,%ymm14
vpermq $3,%ymm13,%ymm15
vblendpd $1,%ymm15,%ymm14,%ymm14
vpermq $144,%ymm13,%ymm13
vpermq $3,%ymm2,%ymm15
vblendpd $1,%ymm15,%ymm13,%ymm13
vpermq $144,%ymm2,%ymm2
vpermq $3,%ymm1,%ymm15
vblendpd $1,%ymm15,%ymm2,%ymm2
vpermq $144,%ymm1,%ymm1
vpermq $3,%ymm0,%ymm15
vblendpd $1,%ymm15,%ymm1,%ymm1
vpermq $144,%ymm0,%ymm0
vpand .Lhigh64x3(%rip),%ymm0,%ymm0
vpand .Lmask52x4(%rip),%ymm3,%ymm3
vpand .Lmask52x4(%rip),%ymm5,%ymm5
vpand .Lmask52x4(%rip),%ymm6,%ymm6
vpand .Lmask52x4(%rip),%ymm7,%ymm7
vpand .Lmask52x4(%rip),%ymm8,%ymm8
vpaddq %ymm0,%ymm3,%ymm3
vpaddq %ymm1,%ymm5,%ymm5
vpaddq %ymm2,%ymm6,%ymm6
vpaddq %ymm13,%ymm7,%ymm7
vpaddq %ymm14,%ymm8,%ymm8
vpcmpgtq .Lmask52x4(%rip),%ymm3,%ymm0
vpcmpgtq .Lmask52x4(%rip),%ymm5,%ymm1
vpcmpgtq .Lmask52x4(%rip),%ymm6,%ymm2
vpcmpgtq .Lmask52x4(%rip),%ymm7,%ymm13
vpcmpgtq .Lmask52x4(%rip),%ymm8,%ymm14
vmovmskpd %ymm0,%r14d
vmovmskpd %ymm1,%r13d
vmovmskpd %ymm2,%r12d
vmovmskpd %ymm13,%r11d
vmovmskpd %ymm14,%r10d
vpcmpeqq .Lmask52x4(%rip),%ymm3,%ymm0
vpcmpeqq .Lmask52x4(%rip),%ymm5,%ymm1
vpcmpeqq .Lmask52x4(%rip),%ymm6,%ymm2
vpcmpeqq .Lmask52x4(%rip),%ymm7,%ymm13
vpcmpeqq .Lmask52x4(%rip),%ymm8,%ymm14
vmovmskpd %ymm0,%r9d
vmovmskpd %ymm1,%r8d
vmovmskpd %ymm2,%ebx
vmovmskpd %ymm13,%ecx
vmovmskpd %ymm14,%edx
shlb $4,%r13b
orb %r13b,%r14b
shlb $4,%r11b
orb %r11b,%r12b
addb %r14b,%r14b
adcb %r12b,%r12b
adcb %r10b,%r10b
shlb $4,%r8b
orb %r8b,%r9b
shlb $4,%cl
orb %cl,%bl
addb %r9b,%r14b
adcb %bl,%r12b
adcb %dl,%r10b
xorb %r9b,%r14b
xorb %bl,%r12b
xorb %dl,%r10b
leaq .Lkmasklut(%rip),%rdx
movb %r14b,%r13b
andq $0xf,%r14
vpsubq .Lmask52x4(%rip),%ymm3,%ymm0
shlq $5,%r14
vmovapd (%rdx,%r14,1),%ymm2
vblendvpd %ymm2,%ymm0,%ymm3,%ymm3
shrb $4,%r13b
andq $0xf,%r13
vpsubq .Lmask52x4(%rip),%ymm5,%ymm0
shlq $5,%r13
vmovapd (%rdx,%r13,1),%ymm2
vblendvpd %ymm2,%ymm0,%ymm5,%ymm5
movb %r12b,%r11b
andq $0xf,%r12
vpsubq .Lmask52x4(%rip),%ymm6,%ymm0
shlq $5,%r12
vmovapd (%rdx,%r12,1),%ymm2
vblendvpd %ymm2,%ymm0,%ymm6,%ymm6
shrb $4,%r11b
andq $0xf,%r11
vpsubq .Lmask52x4(%rip),%ymm7,%ymm0
shlq $5,%r11
vmovapd (%rdx,%r11,1),%ymm2
vblendvpd %ymm2,%ymm0,%ymm7,%ymm7
andq $0xf,%r10
vpsubq .Lmask52x4(%rip),%ymm8,%ymm0
shlq $5,%r10
vmovapd (%rdx,%r10,1),%ymm2
vblendvpd %ymm2,%ymm0,%ymm8,%ymm8
vpand .Lmask52x4(%rip),%ymm3,%ymm3
vpand .Lmask52x4(%rip),%ymm5,%ymm5
vpand .Lmask52x4(%rip),%ymm6,%ymm6
vpand .Lmask52x4(%rip),%ymm7,%ymm7
vpand .Lmask52x4(%rip),%ymm8,%ymm8
vmovdqu %ymm3,0(%rdi)
vmovdqu %ymm5,32(%rdi)
vmovdqu %ymm6,64(%rdi)
vmovdqu %ymm7,96(%rdi)
vmovdqu %ymm8,128(%rdi)
vzeroupper
movq 0(%rsp),%r15
.cfi_restore %r15
movq 8(%rsp),%r14
.cfi_restore %r14
movq 16(%rsp),%r13
.cfi_restore %r13
movq 24(%rsp),%r12
.cfi_restore %r12
movq 32(%rsp),%rbp
.cfi_restore %rbp
movq 40(%rsp),%rbx
.cfi_restore %rbx
leaq 48(%rsp),%rsp
.cfi_adjust_cfa_offset -48
.Lossl_rsaz_amm52x20_x1_avxifma256_epilogue:
.byte 0xf3,0xc3
.cfi_endproc
.size ossl_rsaz_amm52x20_x1_avxifma256, .-ossl_rsaz_amm52x20_x1_avxifma256
.section .rodata
.align 32
.Lmask52x4:
.quad 0xfffffffffffff
.quad 0xfffffffffffff
.quad 0xfffffffffffff
.quad 0xfffffffffffff
.Lhigh64x3:
.quad 0x0
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.Lkmasklut:
.quad 0x0
.quad 0x0
.quad 0x0
.quad 0x0
.quad 0xffffffffffffffff
.quad 0x0
.quad 0x0
.quad 0x0
.quad 0x0
.quad 0xffffffffffffffff
.quad 0x0
.quad 0x0
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.quad 0x0
.quad 0x0
.quad 0x0
.quad 0x0
.quad 0xffffffffffffffff
.quad 0x0
.quad 0xffffffffffffffff
.quad 0x0
.quad 0xffffffffffffffff
.quad 0x0
.quad 0x0
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.quad 0x0
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.quad 0x0
.quad 0x0
.quad 0x0
.quad 0x0
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.quad 0x0
.quad 0x0
.quad 0xffffffffffffffff
.quad 0x0
.quad 0xffffffffffffffff
.quad 0x0
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.quad 0x0
.quad 0xffffffffffffffff
.quad 0x0
.quad 0x0
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.quad 0x0
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.quad 0x0
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.quad 0xffffffffffffffff
.text
.globl ossl_rsaz_amm52x20_x2_avxifma256
.type ossl_rsaz_amm52x20_x2_avxifma256,@function
.align 32
ossl_rsaz_amm52x20_x2_avxifma256:
.cfi_startproc
.byte 243,15,30,250
pushq %rbx
.cfi_adjust_cfa_offset 8
.cfi_offset %rbx,-16
pushq %rbp
.cfi_adjust_cfa_offset 8
.cfi_offset %rbp,-24
pushq %r12
.cfi_adjust_cfa_offset 8
.cfi_offset %r12,-32
pushq %r13
.cfi_adjust_cfa_offset 8
.cfi_offset %r13,-40
pushq %r14
.cfi_adjust_cfa_offset 8
.cfi_offset %r14,-48
pushq %r15
.cfi_adjust_cfa_offset 8
.cfi_offset %r15,-56
.Lossl_rsaz_amm52x20_x2_avxifma256_body:
vpxor %ymm0,%ymm0,%ymm0
vmovapd %ymm0,%ymm3
vmovapd %ymm0,%ymm5
vmovapd %ymm0,%ymm6
vmovapd %ymm0,%ymm7
vmovapd %ymm0,%ymm8
vmovapd %ymm0,%ymm4
vmovapd %ymm0,%ymm9
vmovapd %ymm0,%ymm10
vmovapd %ymm0,%ymm11
vmovapd %ymm0,%ymm12
xorl %r9d,%r9d
xorl %r15d,%r15d
movq %rdx,%r11
movq $0xfffffffffffff,%rax
movl $20,%ebx
.align 32
.Lloop20:
movq 0(%r11),%r13
vpbroadcastq 0(%r11),%ymm1
movq 0(%rsi),%rdx
mulxq %r13,%r13,%r12
addq %r13,%r9
movq %r12,%r10
adcq $0,%r10
movq (%r8),%r13
imulq %r9,%r13
andq %rax,%r13
vmovq %r13,%xmm2
vpbroadcastq %xmm2,%ymm2
movq 0(%rcx),%rdx
mulxq %r13,%r13,%r12
addq %r13,%r9
adcq %r12,%r10
shrq $52,%r9
salq $12,%r10
orq %r10,%r9
leaq -168(%rsp),%rsp
{vex} vpmadd52luq 0(%rsi),%ymm1,%ymm3
{vex} vpmadd52luq 32(%rsi),%ymm1,%ymm5
{vex} vpmadd52luq 64(%rsi),%ymm1,%ymm6
{vex} vpmadd52luq 96(%rsi),%ymm1,%ymm7
{vex} vpmadd52luq 128(%rsi),%ymm1,%ymm8
{vex} vpmadd52luq 0(%rcx),%ymm2,%ymm3
{vex} vpmadd52luq 32(%rcx),%ymm2,%ymm5
{vex} vpmadd52luq 64(%rcx),%ymm2,%ymm6
{vex} vpmadd52luq 96(%rcx),%ymm2,%ymm7
{vex} vpmadd52luq 128(%rcx),%ymm2,%ymm8
vmovdqu %ymm3,0(%rsp)
vmovdqu %ymm5,32(%rsp)
vmovdqu %ymm6,64(%rsp)
vmovdqu %ymm7,96(%rsp)
vmovdqu %ymm8,128(%rsp)
movq $0,160(%rsp)
vmovdqu 8(%rsp),%ymm3
vmovdqu 40(%rsp),%ymm5
vmovdqu 72(%rsp),%ymm6
vmovdqu 104(%rsp),%ymm7
vmovdqu 136(%rsp),%ymm8
addq 8(%rsp),%r9
{vex} vpmadd52huq 0(%rsi),%ymm1,%ymm3
{vex} vpmadd52huq 32(%rsi),%ymm1,%ymm5
{vex} vpmadd52huq 64(%rsi),%ymm1,%ymm6
{vex} vpmadd52huq 96(%rsi),%ymm1,%ymm7
{vex} vpmadd52huq 128(%rsi),%ymm1,%ymm8
{vex} vpmadd52huq 0(%rcx),%ymm2,%ymm3
{vex} vpmadd52huq 32(%rcx),%ymm2,%ymm5
{vex} vpmadd52huq 64(%rcx),%ymm2,%ymm6
{vex} vpmadd52huq 96(%rcx),%ymm2,%ymm7
{vex} vpmadd52huq 128(%rcx),%ymm2,%ymm8
leaq 168(%rsp),%rsp
movq 160(%r11),%r13
vpbroadcastq 160(%r11),%ymm1
movq 160(%rsi),%rdx
mulxq %r13,%r13,%r12
addq %r13,%r15
movq %r12,%r10
adcq $0,%r10
movq 8(%r8),%r13
imulq %r15,%r13
andq %rax,%r13
vmovq %r13,%xmm2
vpbroadcastq %xmm2,%ymm2
movq 160(%rcx),%rdx
mulxq %r13,%r13,%r12
addq %r13,%r15
adcq %r12,%r10
shrq $52,%r15
salq $12,%r10
orq %r10,%r15
leaq -168(%rsp),%rsp
{vex} vpmadd52luq 160(%rsi),%ymm1,%ymm4
{vex} vpmadd52luq 192(%rsi),%ymm1,%ymm9
{vex} vpmadd52luq 224(%rsi),%ymm1,%ymm10
{vex} vpmadd52luq 256(%rsi),%ymm1,%ymm11
{vex} vpmadd52luq 288(%rsi),%ymm1,%ymm12
{vex} vpmadd52luq 160(%rcx),%ymm2,%ymm4
{vex} vpmadd52luq 192(%rcx),%ymm2,%ymm9
{vex} vpmadd52luq 224(%rcx),%ymm2,%ymm10
{vex} vpmadd52luq 256(%rcx),%ymm2,%ymm11
{vex} vpmadd52luq 288(%rcx),%ymm2,%ymm12
vmovdqu %ymm4,0(%rsp)
vmovdqu %ymm9,32(%rsp)
vmovdqu %ymm10,64(%rsp)
vmovdqu %ymm11,96(%rsp)
vmovdqu %ymm12,128(%rsp)
movq $0,160(%rsp)
vmovdqu 8(%rsp),%ymm4
vmovdqu 40(%rsp),%ymm9
vmovdqu 72(%rsp),%ymm10
vmovdqu 104(%rsp),%ymm11
vmovdqu 136(%rsp),%ymm12
addq 8(%rsp),%r15
{vex} vpmadd52huq 160(%rsi),%ymm1,%ymm4
{vex} vpmadd52huq 192(%rsi),%ymm1,%ymm9
{vex} vpmadd52huq 224(%rsi),%ymm1,%ymm10
{vex} vpmadd52huq 256(%rsi),%ymm1,%ymm11
{vex} vpmadd52huq 288(%rsi),%ymm1,%ymm12
{vex} vpmadd52huq 160(%rcx),%ymm2,%ymm4
{vex} vpmadd52huq 192(%rcx),%ymm2,%ymm9
{vex} vpmadd52huq 224(%rcx),%ymm2,%ymm10
{vex} vpmadd52huq 256(%rcx),%ymm2,%ymm11
{vex} vpmadd52huq 288(%rcx),%ymm2,%ymm12
leaq 168(%rsp),%rsp
leaq 8(%r11),%r11
decl %ebx
jne .Lloop20
vmovq %r9,%xmm0
vpbroadcastq %xmm0,%ymm0
vpblendd $3,%ymm0,%ymm3,%ymm3
vpsrlq $52,%ymm3,%ymm0
vpsrlq $52,%ymm5,%ymm1
vpsrlq $52,%ymm6,%ymm2
vpsrlq $52,%ymm7,%ymm13
vpsrlq $52,%ymm8,%ymm14
vpermq $144,%ymm14,%ymm14
vpermq $3,%ymm13,%ymm15
vblendpd $1,%ymm15,%ymm14,%ymm14
vpermq $144,%ymm13,%ymm13
vpermq $3,%ymm2,%ymm15
vblendpd $1,%ymm15,%ymm13,%ymm13
vpermq $144,%ymm2,%ymm2
vpermq $3,%ymm1,%ymm15
vblendpd $1,%ymm15,%ymm2,%ymm2
vpermq $144,%ymm1,%ymm1
vpermq $3,%ymm0,%ymm15
vblendpd $1,%ymm15,%ymm1,%ymm1
vpermq $144,%ymm0,%ymm0
vpand .Lhigh64x3(%rip),%ymm0,%ymm0
vpand .Lmask52x4(%rip),%ymm3,%ymm3
vpand .Lmask52x4(%rip),%ymm5,%ymm5
vpand .Lmask52x4(%rip),%ymm6,%ymm6
vpand .Lmask52x4(%rip),%ymm7,%ymm7
vpand .Lmask52x4(%rip),%ymm8,%ymm8
vpaddq %ymm0,%ymm3,%ymm3
vpaddq %ymm1,%ymm5,%ymm5
vpaddq %ymm2,%ymm6,%ymm6
vpaddq %ymm13,%ymm7,%ymm7
vpaddq %ymm14,%ymm8,%ymm8
vpcmpgtq .Lmask52x4(%rip),%ymm3,%ymm0
vpcmpgtq .Lmask52x4(%rip),%ymm5,%ymm1
vpcmpgtq .Lmask52x4(%rip),%ymm6,%ymm2
vpcmpgtq .Lmask52x4(%rip),%ymm7,%ymm13
vpcmpgtq .Lmask52x4(%rip),%ymm8,%ymm14
vmovmskpd %ymm0,%r14d
vmovmskpd %ymm1,%r13d
vmovmskpd %ymm2,%r12d
vmovmskpd %ymm13,%r11d
vmovmskpd %ymm14,%r10d
vpcmpeqq .Lmask52x4(%rip),%ymm3,%ymm0
vpcmpeqq .Lmask52x4(%rip),%ymm5,%ymm1
vpcmpeqq .Lmask52x4(%rip),%ymm6,%ymm2
vpcmpeqq .Lmask52x4(%rip),%ymm7,%ymm13
vpcmpeqq .Lmask52x4(%rip),%ymm8,%ymm14
vmovmskpd %ymm0,%r9d
vmovmskpd %ymm1,%r8d
vmovmskpd %ymm2,%ebx
vmovmskpd %ymm13,%ecx
vmovmskpd %ymm14,%edx
shlb $4,%r13b
orb %r13b,%r14b
shlb $4,%r11b
orb %r11b,%r12b
addb %r14b,%r14b
adcb %r12b,%r12b
adcb %r10b,%r10b
shlb $4,%r8b
orb %r8b,%r9b
shlb $4,%cl
orb %cl,%bl
addb %r9b,%r14b
adcb %bl,%r12b
adcb %dl,%r10b
xorb %r9b,%r14b
xorb %bl,%r12b
xorb %dl,%r10b
leaq .Lkmasklut(%rip),%rdx
movb %r14b,%r13b
andq $0xf,%r14
vpsubq .Lmask52x4(%rip),%ymm3,%ymm0
shlq $5,%r14
vmovapd (%rdx,%r14,1),%ymm2
vblendvpd %ymm2,%ymm0,%ymm3,%ymm3
shrb $4,%r13b
andq $0xf,%r13
vpsubq .Lmask52x4(%rip),%ymm5,%ymm0
shlq $5,%r13
vmovapd (%rdx,%r13,1),%ymm2
vblendvpd %ymm2,%ymm0,%ymm5,%ymm5
movb %r12b,%r11b
andq $0xf,%r12
vpsubq .Lmask52x4(%rip),%ymm6,%ymm0
shlq $5,%r12
vmovapd (%rdx,%r12,1),%ymm2
vblendvpd %ymm2,%ymm0,%ymm6,%ymm6
shrb $4,%r11b
andq $0xf,%r11
vpsubq .Lmask52x4(%rip),%ymm7,%ymm0
shlq $5,%r11
vmovapd (%rdx,%r11,1),%ymm2
vblendvpd %ymm2,%ymm0,%ymm7,%ymm7
andq $0xf,%r10
vpsubq .Lmask52x4(%rip),%ymm8,%ymm0
shlq $5,%r10
vmovapd (%rdx,%r10,1),%ymm2
vblendvpd %ymm2,%ymm0,%ymm8,%ymm8
vpand .Lmask52x4(%rip),%ymm3,%ymm3
vpand .Lmask52x4(%rip),%ymm5,%ymm5
vpand .Lmask52x4(%rip),%ymm6,%ymm6
vpand .Lmask52x4(%rip),%ymm7,%ymm7
vpand .Lmask52x4(%rip),%ymm8,%ymm8
vmovq %r15,%xmm0
vpbroadcastq %xmm0,%ymm0
vpblendd $3,%ymm0,%ymm4,%ymm4
vpsrlq $52,%ymm4,%ymm0
vpsrlq $52,%ymm9,%ymm1
vpsrlq $52,%ymm10,%ymm2
vpsrlq $52,%ymm11,%ymm13
vpsrlq $52,%ymm12,%ymm14
vpermq $144,%ymm14,%ymm14
vpermq $3,%ymm13,%ymm15
vblendpd $1,%ymm15,%ymm14,%ymm14
vpermq $144,%ymm13,%ymm13
vpermq $3,%ymm2,%ymm15
vblendpd $1,%ymm15,%ymm13,%ymm13
vpermq $144,%ymm2,%ymm2
vpermq $3,%ymm1,%ymm15
vblendpd $1,%ymm15,%ymm2,%ymm2
vpermq $144,%ymm1,%ymm1
vpermq $3,%ymm0,%ymm15
vblendpd $1,%ymm15,%ymm1,%ymm1
vpermq $144,%ymm0,%ymm0
vpand .Lhigh64x3(%rip),%ymm0,%ymm0
vpand .Lmask52x4(%rip),%ymm4,%ymm4
vpand .Lmask52x4(%rip),%ymm9,%ymm9
vpand .Lmask52x4(%rip),%ymm10,%ymm10
vpand .Lmask52x4(%rip),%ymm11,%ymm11
vpand .Lmask52x4(%rip),%ymm12,%ymm12
vpaddq %ymm0,%ymm4,%ymm4
vpaddq %ymm1,%ymm9,%ymm9
vpaddq %ymm2,%ymm10,%ymm10
vpaddq %ymm13,%ymm11,%ymm11
vpaddq %ymm14,%ymm12,%ymm12
vpcmpgtq .Lmask52x4(%rip),%ymm4,%ymm0
vpcmpgtq .Lmask52x4(%rip),%ymm9,%ymm1
vpcmpgtq .Lmask52x4(%rip),%ymm10,%ymm2
vpcmpgtq .Lmask52x4(%rip),%ymm11,%ymm13
vpcmpgtq .Lmask52x4(%rip),%ymm12,%ymm14
vmovmskpd %ymm0,%r14d
vmovmskpd %ymm1,%r13d
vmovmskpd %ymm2,%r12d
vmovmskpd %ymm13,%r11d
vmovmskpd %ymm14,%r10d
vpcmpeqq .Lmask52x4(%rip),%ymm4,%ymm0
vpcmpeqq .Lmask52x4(%rip),%ymm9,%ymm1
vpcmpeqq .Lmask52x4(%rip),%ymm10,%ymm2
vpcmpeqq .Lmask52x4(%rip),%ymm11,%ymm13
vpcmpeqq .Lmask52x4(%rip),%ymm12,%ymm14
vmovmskpd %ymm0,%r9d
vmovmskpd %ymm1,%r8d
vmovmskpd %ymm2,%ebx
vmovmskpd %ymm13,%ecx
vmovmskpd %ymm14,%edx
shlb $4,%r13b
orb %r13b,%r14b
shlb $4,%r11b
orb %r11b,%r12b
addb %r14b,%r14b
adcb %r12b,%r12b
adcb %r10b,%r10b
shlb $4,%r8b
orb %r8b,%r9b
shlb $4,%cl
orb %cl,%bl
addb %r9b,%r14b
adcb %bl,%r12b
adcb %dl,%r10b
xorb %r9b,%r14b
xorb %bl,%r12b
xorb %dl,%r10b
leaq .Lkmasklut(%rip),%rdx
movb %r14b,%r13b
andq $0xf,%r14
vpsubq .Lmask52x4(%rip),%ymm4,%ymm0
shlq $5,%r14
vmovapd (%rdx,%r14,1),%ymm2
vblendvpd %ymm2,%ymm0,%ymm4,%ymm4
shrb $4,%r13b
andq $0xf,%r13
vpsubq .Lmask52x4(%rip),%ymm9,%ymm0
shlq $5,%r13
vmovapd (%rdx,%r13,1),%ymm2
vblendvpd %ymm2,%ymm0,%ymm9,%ymm9
movb %r12b,%r11b
andq $0xf,%r12
vpsubq .Lmask52x4(%rip),%ymm10,%ymm0
shlq $5,%r12
vmovapd (%rdx,%r12,1),%ymm2
vblendvpd %ymm2,%ymm0,%ymm10,%ymm10
shrb $4,%r11b
andq $0xf,%r11
vpsubq .Lmask52x4(%rip),%ymm11,%ymm0
shlq $5,%r11
vmovapd (%rdx,%r11,1),%ymm2
vblendvpd %ymm2,%ymm0,%ymm11,%ymm11
andq $0xf,%r10
vpsubq .Lmask52x4(%rip),%ymm12,%ymm0
shlq $5,%r10
vmovapd (%rdx,%r10,1),%ymm2
vblendvpd %ymm2,%ymm0,%ymm12,%ymm12
vpand .Lmask52x4(%rip),%ymm4,%ymm4
vpand .Lmask52x4(%rip),%ymm9,%ymm9
vpand .Lmask52x4(%rip),%ymm10,%ymm10
vpand .Lmask52x4(%rip),%ymm11,%ymm11
vpand .Lmask52x4(%rip),%ymm12,%ymm12
vmovdqu %ymm3,0(%rdi)
vmovdqu %ymm5,32(%rdi)
vmovdqu %ymm6,64(%rdi)
vmovdqu %ymm7,96(%rdi)
vmovdqu %ymm8,128(%rdi)
vmovdqu %ymm4,160(%rdi)
vmovdqu %ymm9,192(%rdi)
vmovdqu %ymm10,224(%rdi)
vmovdqu %ymm11,256(%rdi)
vmovdqu %ymm12,288(%rdi)
vzeroupper
movq 0(%rsp),%r15
.cfi_restore %r15
movq 8(%rsp),%r14
.cfi_restore %r14
movq 16(%rsp),%r13
.cfi_restore %r13
movq 24(%rsp),%r12
.cfi_restore %r12
movq 32(%rsp),%rbp
.cfi_restore %rbp
movq 40(%rsp),%rbx
.cfi_restore %rbx
leaq 48(%rsp),%rsp
.cfi_adjust_cfa_offset -48
.Lossl_rsaz_amm52x20_x2_avxifma256_epilogue:
.byte 0xf3,0xc3
.cfi_endproc
.size ossl_rsaz_amm52x20_x2_avxifma256, .-ossl_rsaz_amm52x20_x2_avxifma256
.text
.align 32
.globl ossl_extract_multiplier_2x20_win5_avx
.type ossl_extract_multiplier_2x20_win5_avx,@function
ossl_extract_multiplier_2x20_win5_avx:
.cfi_startproc
.byte 243,15,30,250
vmovapd .Lones(%rip),%ymm14
vmovq %rdx,%xmm10
vpbroadcastq %xmm10,%ymm12
vmovq %rcx,%xmm10
vpbroadcastq %xmm10,%ymm13
leaq 10240(%rsi),%rax
vpxor %xmm0,%xmm0,%xmm0
vmovapd %ymm0,%ymm11
vmovapd %ymm0,%ymm1
vmovapd %ymm0,%ymm2
vmovapd %ymm0,%ymm3
vmovapd %ymm0,%ymm4
vmovapd %ymm0,%ymm5
vmovapd %ymm0,%ymm6
vmovapd %ymm0,%ymm7
vmovapd %ymm0,%ymm8
vmovapd %ymm0,%ymm9
.align 32
.Lloop:
vpcmpeqq %ymm11,%ymm12,%ymm15
vmovdqu 0(%rsi),%ymm10
vblendvpd %ymm15,%ymm10,%ymm0,%ymm0
vmovdqu 32(%rsi),%ymm10
vblendvpd %ymm15,%ymm10,%ymm1,%ymm1
vmovdqu 64(%rsi),%ymm10
vblendvpd %ymm15,%ymm10,%ymm2,%ymm2
vmovdqu 96(%rsi),%ymm10
vblendvpd %ymm15,%ymm10,%ymm3,%ymm3
vmovdqu 128(%rsi),%ymm10
vblendvpd %ymm15,%ymm10,%ymm4,%ymm4
vpcmpeqq %ymm11,%ymm13,%ymm15
vmovdqu 160(%rsi),%ymm10
vblendvpd %ymm15,%ymm10,%ymm5,%ymm5
vmovdqu 192(%rsi),%ymm10
vblendvpd %ymm15,%ymm10,%ymm6,%ymm6
vmovdqu 224(%rsi),%ymm10
vblendvpd %ymm15,%ymm10,%ymm7,%ymm7
vmovdqu 256(%rsi),%ymm10
vblendvpd %ymm15,%ymm10,%ymm8,%ymm8
vmovdqu 288(%rsi),%ymm10
vblendvpd %ymm15,%ymm10,%ymm9,%ymm9
vpaddq %ymm14,%ymm11,%ymm11
addq $320,%rsi
cmpq %rsi,%rax
jne .Lloop
vmovdqu %ymm0,0(%rdi)
vmovdqu %ymm1,32(%rdi)
vmovdqu %ymm2,64(%rdi)
vmovdqu %ymm3,96(%rdi)
vmovdqu %ymm4,128(%rdi)
vmovdqu %ymm5,160(%rdi)
vmovdqu %ymm6,192(%rdi)
vmovdqu %ymm7,224(%rdi)
vmovdqu %ymm8,256(%rdi)
vmovdqu %ymm9,288(%rdi)
vzeroupper
.byte 0xf3,0xc3
.cfi_endproc
.size ossl_extract_multiplier_2x20_win5_avx, .-ossl_extract_multiplier_2x20_win5_avx
.section .rodata
.align 32
.Lones:
.quad 1,1,1,1
.Lzeros:
.quad 0,0,0,0
.section ".note.gnu.property", "a"
.p2align 3
.long 1f - 0f
.long 4f - 1f
.long 5
0:
# "GNU" encoded with .byte, since .asciz isn't supported
.byte 0x47
.byte 0x4e
.byte 0x55
.byte 0
1:
.p2align 3
.long 0xc0000002
.long 3f - 2f
2:
.long 3
3:
.p2align 3
4:
