/* SPDX-License-Identifier: GPL-2.0 OR MIT */
/*
 * Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
 * Copyright (C) 2017-2019 Samuel Neves <sneves@dei.uc.pt>. All Rights Reserved.
 */

#include <linux/linkage.h>

.section .rodata.cst32.iv, "aM", @progbits, 32
.align 32
.Liv:
        .octa 0xA54FF53A3C6EF372BB67AE856A09E667
        .octa 0x5BE0CD191F83D9AB9B05688C510E527F

.section .rodata.cst16.ror16, "aM", @progbits, 16
.align 16
.Lror16:
        .octa 0x0D0C0F0E09080B0A0504070601000302

.section .rodata.cst16.ror8, "aM", @progbits, 16
.align 16
.Lror8:
        .octa 0x0C0F0E0D080B0A090407060500030201

.section .rodata.cst64.sigma, "aM", @progbits, 160
.align 64
.Lsigma:
.byte  0,  2,  4,  6,  1,  3,  5,  7, 14,  8, 10, 12, 15,  9, 11, 13
.byte 14,  4,  9, 13, 10,  8, 15,  6,  5,  1,  0, 11,  3, 12,  2,  7
.byte 11, 12,  5, 15,  8,  0,  2, 13,  9, 10,  3,  7,  4, 14,  6,  1
.byte  7,  3, 13, 11,  9,  1, 12, 14, 15,  2,  5,  4,  8,  6, 10,  0
.byte  9,  5,  2, 10,  0,  7,  4, 15,  3, 14, 11,  6, 13,  1, 12,  8
.byte  2,  6,  0,  8, 12, 10, 11,  3,  1,  4,  7, 15,  9, 13,  5, 14
.byte 12,  1, 14,  4,  5, 15, 13, 10,  8,  0,  6,  9, 11,  7,  3,  2
.byte 13,  7, 12,  3, 11, 14,  1,  9,  2,  5, 15,  8, 10,  0,  4,  6
.byte  6, 14, 11,  0, 15,  9,  3,  8, 10, 12, 13,  1,  5,  2,  7,  4
.byte 10,  8,  7,  1,  2,  4,  6,  5, 13, 15,  9,  3,  0, 11, 14, 12

.section .rodata.cst64.sigma2, "aM", @progbits, 160
.align 64
.Lsigma2:
.byte  0,  2,  4,  6,  1,  3,  5,  7, 14,  8, 10, 12, 15,  9, 11, 13
.byte  8,  2, 13, 15, 10,  9, 12,  3,  6,  4,  0, 14,  5, 11,  1,  7
.byte 11, 13,  8,  6,  5, 10, 14,  3,  2,  4, 12, 15,  1,  0,  7,  9
.byte 11, 10,  7,  0,  8, 15,  1, 13,  3,  6,  2, 12,  4, 14,  9,  5
.byte  4, 10,  9, 14, 15,  0, 11,  8,  1,  7,  3, 13,  2,  5,  6, 12
.byte  2, 11,  4, 15, 14,  3, 10,  8, 13,  6,  5,  7,  0, 12,  1,  9
.byte  4,  8, 15,  9, 14, 11, 13,  5,  3,  2,  1, 12,  6, 10,  7,  0
.byte  6, 13,  0, 14, 12,  2,  1, 11, 15,  4,  5,  8,  7,  9,  3, 10
.byte 15,  5,  4, 13, 10,  7,  3, 11, 12,  2,  0,  6,  9,  8,  1, 14
.byte  8,  7, 14, 11, 13, 15,  0, 12, 10,  4,  5,  6,  3,  2,  1,  9

#define CTX             %rdi
#define DATA            %rsi
#define NBLOCKS         %rdx
#define INC             %ecx

.text
//
// void blake2s_compress_ssse3(struct blake2s_ctx *ctx,
//                             const u8 *data, size_t nblocks, u32 inc);
//
// Only the first three fields of struct blake2s_ctx are used:
//      u32 h[8];       (inout)
//      u32 t[2];       (inout)
//      u32 f[2];       (in)
//
SYM_FUNC_START(blake2s_compress_ssse3)
        movdqu          (CTX),%xmm0             // Load h[0..3]
        movdqu          16(CTX),%xmm1           // Load h[4..7]
        movdqa          .Lror16(%rip),%xmm12
        movdqa          .Lror8(%rip),%xmm13
        movdqu          32(CTX),%xmm14          // Load t and f
        movd            INC,%xmm15              // Load inc
        leaq            .Lsigma+160(%rip),%r8
        jmp             .Lssse3_mainloop

        .align          32
.Lssse3_mainloop:
        // Main loop: each iteration processes one 64-byte block.
        movdqa          %xmm0,%xmm10            // Save h[0..3] and let v[0..3] = h[0..3]
        movdqa          %xmm1,%xmm11            // Save h[4..7] and let v[4..7] = h[4..7]
        paddq           %xmm15,%xmm14           // t += inc (64-bit addition)
        movdqa          .Liv(%rip),%xmm2        // v[8..11] = iv[0..3]
        movdqa          %xmm14,%xmm3
        pxor            .Liv+16(%rip),%xmm3     // v[12..15] = iv[4..7] ^ [t, f]
        leaq            .Lsigma(%rip),%rcx

.Lssse3_roundloop:
        // Round loop: each iteration does 1 round (of 10 rounds total).
        movzbl          (%rcx),%eax
        movd            (DATA,%rax,4),%xmm4
        movzbl          1(%rcx),%eax
        movd            (DATA,%rax,4),%xmm5
        movzbl          2(%rcx),%eax
        movd            (DATA,%rax,4),%xmm6
        movzbl          3(%rcx),%eax
        movd            (DATA,%rax,4),%xmm7
        punpckldq       %xmm5,%xmm4
        punpckldq       %xmm7,%xmm6
        punpcklqdq      %xmm6,%xmm4
        paddd           %xmm4,%xmm0
        paddd           %xmm1,%xmm0
        pxor            %xmm0,%xmm3
        pshufb          %xmm12,%xmm3
        paddd           %xmm3,%xmm2
        pxor            %xmm2,%xmm1
        movdqa          %xmm1,%xmm8
        psrld           $12,%xmm1
        pslld           $20,%xmm8
        por             %xmm8,%xmm1
        movzbl          4(%rcx),%eax
        movd            (DATA,%rax,4),%xmm5
        movzbl          5(%rcx),%eax
        movd            (DATA,%rax,4),%xmm6
        movzbl          6(%rcx),%eax
        movd            (DATA,%rax,4),%xmm7
        movzbl          7(%rcx),%eax
        movd            (DATA,%rax,4),%xmm4
        punpckldq       %xmm6,%xmm5
        punpckldq       %xmm4,%xmm7
        punpcklqdq      %xmm7,%xmm5
        paddd           %xmm5,%xmm0
        paddd           %xmm1,%xmm0
        pxor            %xmm0,%xmm3
        pshufb          %xmm13,%xmm3
        paddd           %xmm3,%xmm2
        pxor            %xmm2,%xmm1
        movdqa          %xmm1,%xmm8
        psrld           $7,%xmm1
        pslld           $25,%xmm8
        por             %xmm8,%xmm1
        pshufd          $0x93,%xmm0,%xmm0
        pshufd          $0x4e,%xmm3,%xmm3
        pshufd          $0x39,%xmm2,%xmm2
        movzbl          8(%rcx),%eax
        movd            (DATA,%rax,4),%xmm6
        movzbl          9(%rcx),%eax
        movd            (DATA,%rax,4),%xmm7
        movzbl          10(%rcx),%eax
        movd            (DATA,%rax,4),%xmm4
        movzbl          11(%rcx),%eax
        movd            (DATA,%rax,4),%xmm5
        punpckldq       %xmm7,%xmm6
        punpckldq       %xmm5,%xmm4
        punpcklqdq      %xmm4,%xmm6
        paddd           %xmm6,%xmm0
        paddd           %xmm1,%xmm0
        pxor            %xmm0,%xmm3
        pshufb          %xmm12,%xmm3
        paddd           %xmm3,%xmm2
        pxor            %xmm2,%xmm1
        movdqa          %xmm1,%xmm8
        psrld           $12,%xmm1
        pslld           $20,%xmm8
        por             %xmm8,%xmm1
        movzbl          12(%rcx),%eax
        movd            (DATA,%rax,4),%xmm7
        movzbl          13(%rcx),%eax
        movd            (DATA,%rax,4),%xmm4
        movzbl          14(%rcx),%eax
        movd            (DATA,%rax,4),%xmm5
        movzbl          15(%rcx),%eax
        movd            (DATA,%rax,4),%xmm6
        punpckldq       %xmm4,%xmm7
        punpckldq       %xmm6,%xmm5
        punpcklqdq      %xmm5,%xmm7
        paddd           %xmm7,%xmm0
        paddd           %xmm1,%xmm0
        pxor            %xmm0,%xmm3
        pshufb          %xmm13,%xmm3
        paddd           %xmm3,%xmm2
        pxor            %xmm2,%xmm1
        movdqa          %xmm1,%xmm8
        psrld           $7,%xmm1
        pslld           $25,%xmm8
        por             %xmm8,%xmm1
        pshufd          $0x39,%xmm0,%xmm0
        pshufd          $0x4e,%xmm3,%xmm3
        pshufd          $0x93,%xmm2,%xmm2
        addq            $16,%rcx
        cmpq            %r8,%rcx
        jnz             .Lssse3_roundloop

        // Compute the new h: h[0..7] ^= v[0..7] ^ v[8..15]
        pxor            %xmm2,%xmm0
        pxor            %xmm3,%xmm1
        pxor            %xmm10,%xmm0
        pxor            %xmm11,%xmm1
        addq            $64,DATA
        decq            NBLOCKS
        jnz             .Lssse3_mainloop

        movdqu          %xmm0,(CTX)             // Store new h[0..3]
        movdqu          %xmm1,16(CTX)           // Store new h[4..7]
        movq            %xmm14,32(CTX)          // Store new t (f is unchanged)
        RET
SYM_FUNC_END(blake2s_compress_ssse3)

//
// void blake2s_compress_avx512(struct blake2s_ctx *ctx,
//                              const u8 *data, size_t nblocks, u32 inc);
//
// Only the first three fields of struct blake2s_ctx are used:
//      u32 h[8];       (inout)
//      u32 t[2];       (inout)
//      u32 f[2];       (in)
//
SYM_FUNC_START(blake2s_compress_avx512)
        vmovdqu         (CTX),%xmm0             // Load h[0..3]
        vmovdqu         16(CTX),%xmm1           // Load h[4..7]
        vmovdqu         32(CTX),%xmm4           // Load t and f
        vmovd           INC,%xmm5               // Load inc
        vmovdqa         .Liv(%rip),%xmm14       // Load iv[0..3]
        vmovdqa         .Liv+16(%rip),%xmm15    // Load iv[4..7]
        jmp             .Lavx512_mainloop

        .align          32
.Lavx512_mainloop:
        // Main loop: each iteration processes one 64-byte block.
        vmovdqa         %xmm0,%xmm10            // Save h[0..3] and let v[0..3] = h[0..3]
        vmovdqa         %xmm1,%xmm11            // Save h[4..7] and let v[4..7] = h[4..7]
        vpaddq          %xmm5,%xmm4,%xmm4       // t += inc (64-bit addition)
        vmovdqa         %xmm14,%xmm2            // v[8..11] = iv[0..3]
        vpxor           %xmm15,%xmm4,%xmm3      // v[12..15] = iv[4..7] ^ [t, f]
        vmovdqu         (DATA),%ymm6            // Load first 8 data words
        vmovdqu         32(DATA),%ymm7          // Load second 8 data words
        addq            $64,DATA
        leaq            .Lsigma2(%rip),%rax
        movb            $10,%cl                 // Set num rounds remaining

.Lavx512_roundloop:
        // Round loop: each iteration does 1 round (of 10 rounds total).
        vpmovzxbd       (%rax),%ymm8
        vpmovzxbd       8(%rax),%ymm9
        addq            $16,%rax
        vpermi2d        %ymm7,%ymm6,%ymm8
        vpermi2d        %ymm7,%ymm6,%ymm9
        vmovdqa         %ymm8,%ymm6
        vmovdqa         %ymm9,%ymm7
        vpaddd          %xmm8,%xmm0,%xmm0
        vpaddd          %xmm1,%xmm0,%xmm0
        vpxor           %xmm0,%xmm3,%xmm3
        vprord          $16,%xmm3,%xmm3
        vpaddd          %xmm3,%xmm2,%xmm2
        vpxor           %xmm2,%xmm1,%xmm1
        vprord          $12,%xmm1,%xmm1
        vextracti128    $1,%ymm8,%xmm8
        vpaddd          %xmm8,%xmm0,%xmm0
        vpaddd          %xmm1,%xmm0,%xmm0
        vpxor           %xmm0,%xmm3,%xmm3
        vprord          $8,%xmm3,%xmm3
        vpaddd          %xmm3,%xmm2,%xmm2
        vpxor           %xmm2,%xmm1,%xmm1
        vprord          $7,%xmm1,%xmm1
        vpshufd         $0x93,%xmm0,%xmm0
        vpshufd         $0x4e,%xmm3,%xmm3
        vpshufd         $0x39,%xmm2,%xmm2
        vpaddd          %xmm9,%xmm0,%xmm0
        vpaddd          %xmm1,%xmm0,%xmm0
        vpxor           %xmm0,%xmm3,%xmm3
        vprord          $16,%xmm3,%xmm3
        vpaddd          %xmm3,%xmm2,%xmm2
        vpxor           %xmm2,%xmm1,%xmm1
        vprord          $12,%xmm1,%xmm1
        vextracti128    $1,%ymm9,%xmm9
        vpaddd          %xmm9,%xmm0,%xmm0
        vpaddd          %xmm1,%xmm0,%xmm0
        vpxor           %xmm0,%xmm3,%xmm3
        vprord          $8,%xmm3,%xmm3
        vpaddd          %xmm3,%xmm2,%xmm2
        vpxor           %xmm2,%xmm1,%xmm1
        vprord          $7,%xmm1,%xmm1
        vpshufd         $0x39,%xmm0,%xmm0
        vpshufd         $0x4e,%xmm3,%xmm3
        vpshufd         $0x93,%xmm2,%xmm2
        decb            %cl
        jne             .Lavx512_roundloop

        // Compute the new h: h[0..7] ^= v[0..7] ^ v[8..15]
        vpternlogd      $0x96,%xmm10,%xmm2,%xmm0
        vpternlogd      $0x96,%xmm11,%xmm3,%xmm1
        decq            NBLOCKS
        jne             .Lavx512_mainloop

        vmovdqu         %xmm0,(CTX)             // Store new h[0..3]
        vmovdqu         %xmm1,16(CTX)           // Store new h[4..7]
        vmovq           %xmm4,32(CTX)           // Store new t (f is unchanged)
        vzeroupper
        RET
SYM_FUNC_END(blake2s_compress_avx512)