/* SPDX-License-Identifier: GPL-2.0-or-later */
#
# Accelerated poly1305 implementation for ppc64le.
#
# Copyright 2023- IBM Corp. All rights reserved
#
#===================================================================================
# Written by Danny Tsen <dtsen@us.ibm.com>
#
# Poly1305 - this version mainly using vector/VSX/Scalar
#  - 26 bits limbs
#  - Handle multiple 64 byte blcok.
#
# Block size 16 bytes
# key = (r, s)
# clamp r &= 0x0FFFFFFC0FFFFFFC 0x0FFFFFFC0FFFFFFF
# p = 2^130 - 5
# a += m
# a = (r + a) % p
# a += s
#
# Improve performance by breaking down polynominal to the sum of products with
#     h4 = m1 * r⁴ + m2 * r³ + m3 * r² + m4 * r
#
#  07/22/21 - this revison based on the above sum of products.  Setup r^4, r^3, r^2, r and s3, s2, s1, s0
#             to 9 vectors for multiplications.
#
# setup r^4, r^3, r^2, r vectors
#    vs    [r^1, r^3, r^2, r^4]
#    vs0 = [r0,.....]
#    vs1 = [r1,.....]
#    vs2 = [r2,.....]
#    vs3 = [r3,.....]
#    vs4 = [r4,.....]
#    vs5 = [r1*5,...]
#    vs6 = [r2*5,...]
#    vs7 = [r2*5,...]
#    vs8 = [r4*5,...]
#
#  Each word in a vector consists a member of a "r/s" in [a * r/s].
#
# r0, r4*5, r3*5, r2*5, r1*5;
# r1, r0,   r4*5, r3*5, r2*5;
# r2, r1,   r0,   r4*5, r3*5;
# r3, r2,   r1,   r0,   r4*5;
# r4, r3,   r2,   r1,   r0  ;
#
#
# poly1305_p10le_4blocks( uint8_t *k, uint32_t mlen, uint8_t *m)
#  k = 32 bytes key
#  r3 = k (r, s)
#  r4 = mlen
#  r5 = m
#
#include <asm/ppc_asm.h>
#include <asm/asm-offsets.h>
#include <asm/asm-compat.h>
#include <linux/linkage.h>

.machine "any"

.text

.macro  SAVE_GPR GPR OFFSET FRAME
        std     \GPR,\OFFSET(\FRAME)
.endm

.macro  SAVE_VRS VRS OFFSET FRAME
        li      16, \OFFSET
        stvx    \VRS, 16, \FRAME
.endm

.macro  SAVE_VSX VSX OFFSET FRAME
        li      16, \OFFSET
        stxvx   \VSX, 16, \FRAME
.endm

.macro  RESTORE_GPR GPR OFFSET FRAME
        ld      \GPR,\OFFSET(\FRAME)
.endm

.macro  RESTORE_VRS VRS OFFSET FRAME
        li      16, \OFFSET
        lvx     \VRS, 16, \FRAME
.endm

.macro  RESTORE_VSX VSX OFFSET FRAME
        li      16, \OFFSET
        lxvx    \VSX, 16, \FRAME
.endm

.macro SAVE_REGS
        mflr 0
        std 0, 16(1)
        stdu 1,-752(1)

        SAVE_GPR 14, 112, 1
        SAVE_GPR 15, 120, 1
        SAVE_GPR 16, 128, 1
        SAVE_GPR 17, 136, 1
        SAVE_GPR 18, 144, 1
        SAVE_GPR 19, 152, 1
        SAVE_GPR 20, 160, 1
        SAVE_GPR 21, 168, 1
        SAVE_GPR 22, 176, 1
        SAVE_GPR 23, 184, 1
        SAVE_GPR 24, 192, 1
        SAVE_GPR 25, 200, 1
        SAVE_GPR 26, 208, 1
        SAVE_GPR 27, 216, 1
        SAVE_GPR 28, 224, 1
        SAVE_GPR 29, 232, 1
        SAVE_GPR 30, 240, 1
        SAVE_GPR 31, 248, 1

        addi    9, 1, 256
        SAVE_VRS 20, 0, 9
        SAVE_VRS 21, 16, 9
        SAVE_VRS 22, 32, 9
        SAVE_VRS 23, 48, 9
        SAVE_VRS 24, 64, 9
        SAVE_VRS 25, 80, 9
        SAVE_VRS 26, 96, 9
        SAVE_VRS 27, 112, 9
        SAVE_VRS 28, 128, 9
        SAVE_VRS 29, 144, 9
        SAVE_VRS 30, 160, 9
        SAVE_VRS 31, 176, 9

        SAVE_VSX 14, 192, 9
        SAVE_VSX 15, 208, 9
        SAVE_VSX 16, 224, 9
        SAVE_VSX 17, 240, 9
        SAVE_VSX 18, 256, 9
        SAVE_VSX 19, 272, 9
        SAVE_VSX 20, 288, 9
        SAVE_VSX 21, 304, 9
        SAVE_VSX 22, 320, 9
        SAVE_VSX 23, 336, 9
        SAVE_VSX 24, 352, 9
        SAVE_VSX 25, 368, 9
        SAVE_VSX 26, 384, 9
        SAVE_VSX 27, 400, 9
        SAVE_VSX 28, 416, 9
        SAVE_VSX 29, 432, 9
        SAVE_VSX 30, 448, 9
        SAVE_VSX 31, 464, 9
.endm # SAVE_REGS

.macro RESTORE_REGS
        addi    9, 1, 256
        RESTORE_VRS 20, 0, 9
        RESTORE_VRS 21, 16, 9
        RESTORE_VRS 22, 32, 9
        RESTORE_VRS 23, 48, 9
        RESTORE_VRS 24, 64, 9
        RESTORE_VRS 25, 80, 9
        RESTORE_VRS 26, 96, 9
        RESTORE_VRS 27, 112, 9
        RESTORE_VRS 28, 128, 9
        RESTORE_VRS 29, 144, 9
        RESTORE_VRS 30, 160, 9
        RESTORE_VRS 31, 176, 9

        RESTORE_VSX 14, 192, 9
        RESTORE_VSX 15, 208, 9
        RESTORE_VSX 16, 224, 9
        RESTORE_VSX 17, 240, 9
        RESTORE_VSX 18, 256, 9
        RESTORE_VSX 19, 272, 9
        RESTORE_VSX 20, 288, 9
        RESTORE_VSX 21, 304, 9
        RESTORE_VSX 22, 320, 9
        RESTORE_VSX 23, 336, 9
        RESTORE_VSX 24, 352, 9
        RESTORE_VSX 25, 368, 9
        RESTORE_VSX 26, 384, 9
        RESTORE_VSX 27, 400, 9
        RESTORE_VSX 28, 416, 9
        RESTORE_VSX 29, 432, 9
        RESTORE_VSX 30, 448, 9
        RESTORE_VSX 31, 464, 9

        RESTORE_GPR 14, 112, 1
        RESTORE_GPR 15, 120, 1
        RESTORE_GPR 16, 128, 1
        RESTORE_GPR 17, 136, 1
        RESTORE_GPR 18, 144, 1
        RESTORE_GPR 19, 152, 1
        RESTORE_GPR 20, 160, 1
        RESTORE_GPR 21, 168, 1
        RESTORE_GPR 22, 176, 1
        RESTORE_GPR 23, 184, 1
        RESTORE_GPR 24, 192, 1
        RESTORE_GPR 25, 200, 1
        RESTORE_GPR 26, 208, 1
        RESTORE_GPR 27, 216, 1
        RESTORE_GPR 28, 224, 1
        RESTORE_GPR 29, 232, 1
        RESTORE_GPR 30, 240, 1
        RESTORE_GPR 31, 248, 1

        addi    1, 1, 752
        ld 0, 16(1)
        mtlr 0
.endm # RESTORE_REGS

#
# p[0] = a0*r0 + a1*r4*5 + a2*r3*5 + a3*r2*5 + a4*r1*5;
# p[1] = a0*r1 + a1*r0   + a2*r4*5 + a3*r3*5 + a4*r2*5;
# p[2] = a0*r2 + a1*r1   + a2*r0   + a3*r4*5 + a4*r3*5;
# p[3] = a0*r3 + a1*r2   + a2*r1   + a3*r0   + a4*r4*5;
# p[4] = a0*r4 + a1*r3   + a2*r2   + a3*r1   + a4*r0  ;
#
#    [r^2, r^3, r^1, r^4]
#    [m3,  m2,  m4,  m1]
#
# multiply odd and even words
.macro mul_odd
        vmulouw 14, 4, 26
        vmulouw 10, 5, 3
        vmulouw 11, 6, 2
        vmulouw 12, 7, 1
        vmulouw 13, 8, 0
        vmulouw 15, 4, 27
        vaddudm 14, 14, 10
        vaddudm 14, 14, 11
        vmulouw 10, 5, 26
        vmulouw 11, 6, 3
        vaddudm 14, 14, 12
        vaddudm 14, 14, 13      # x0
        vaddudm 15, 15, 10
        vaddudm 15, 15, 11
        vmulouw 12, 7, 2
        vmulouw 13, 8, 1
        vaddudm 15, 15, 12
        vaddudm 15, 15, 13      # x1
        vmulouw 16, 4, 28
        vmulouw 10, 5, 27
        vmulouw 11, 6, 26
        vaddudm 16, 16, 10
        vaddudm 16, 16, 11
        vmulouw 12, 7, 3
        vmulouw 13, 8, 2
        vaddudm 16, 16, 12
        vaddudm 16, 16, 13      # x2
        vmulouw 17, 4, 29
        vmulouw 10, 5, 28
        vmulouw 11, 6, 27
        vaddudm 17, 17, 10
        vaddudm 17, 17, 11
        vmulouw 12, 7, 26
        vmulouw 13, 8, 3
        vaddudm 17, 17, 12
        vaddudm 17, 17, 13      # x3
        vmulouw 18, 4, 30
        vmulouw 10, 5, 29
        vmulouw 11, 6, 28
        vaddudm 18, 18, 10
        vaddudm 18, 18, 11
        vmulouw 12, 7, 27
        vmulouw 13, 8, 26
        vaddudm 18, 18, 12
        vaddudm 18, 18, 13      # x4
.endm

.macro mul_even
        vmuleuw 9, 4, 26
        vmuleuw 10, 5, 3
        vmuleuw 11, 6, 2
        vmuleuw 12, 7, 1
        vmuleuw 13, 8, 0
        vaddudm 14, 14, 9
        vaddudm 14, 14, 10
        vaddudm 14, 14, 11
        vaddudm 14, 14, 12
        vaddudm 14, 14, 13      # x0

        vmuleuw 9, 4, 27
        vmuleuw 10, 5, 26
        vmuleuw 11, 6, 3
        vmuleuw 12, 7, 2
        vmuleuw 13, 8, 1
        vaddudm 15, 15, 9
        vaddudm 15, 15, 10
        vaddudm 15, 15, 11
        vaddudm 15, 15, 12
        vaddudm 15, 15, 13      # x1

        vmuleuw 9, 4, 28
        vmuleuw 10, 5, 27
        vmuleuw 11, 6, 26
        vmuleuw 12, 7, 3
        vmuleuw 13, 8, 2
        vaddudm 16, 16, 9
        vaddudm 16, 16, 10
        vaddudm 16, 16, 11
        vaddudm 16, 16, 12
        vaddudm 16, 16, 13      # x2

        vmuleuw 9, 4, 29
        vmuleuw 10, 5, 28
        vmuleuw 11, 6, 27
        vmuleuw 12, 7, 26
        vmuleuw 13, 8, 3
        vaddudm 17, 17, 9
        vaddudm 17, 17, 10
        vaddudm 17, 17, 11
        vaddudm 17, 17, 12
        vaddudm 17, 17, 13      # x3

        vmuleuw 9, 4, 30
        vmuleuw 10, 5, 29
        vmuleuw 11, 6, 28
        vmuleuw 12, 7, 27
        vmuleuw 13, 8, 26
        vaddudm 18, 18, 9
        vaddudm 18, 18, 10
        vaddudm 18, 18, 11
        vaddudm 18, 18, 12
        vaddudm 18, 18, 13      # x4
.endm

#
# poly1305_setup_r
#
# setup r^4, r^3, r^2, r vectors
#    [r, r^3, r^2, r^4]
#    vs0 = [r0,...]
#    vs1 = [r1,...]
#    vs2 = [r2,...]
#    vs3 = [r3,...]
#    vs4 = [r4,...]
#    vs5 = [r4*5,...]
#    vs6 = [r3*5,...]
#    vs7 = [r2*5,...]
#    vs8 = [r1*5,...]
#
# r0, r4*5, r3*5, r2*5, r1*5;
# r1, r0,   r4*5, r3*5, r2*5;
# r2, r1,   r0,   r4*5, r3*5;
# r3, r2,   r1,   r0,   r4*5;
# r4, r3,   r2,   r1,   r0  ;
#
.macro poly1305_setup_r

        # save r
        xxlor   26, 58, 58
        xxlor   27, 59, 59
        xxlor   28, 60, 60
        xxlor   29, 61, 61
        xxlor   30, 62, 62

        xxlxor  31, 31, 31

#    [r, r^3, r^2, r^4]
        # compute r^2
        vmr     4, 26
        vmr     5, 27
        vmr     6, 28
        vmr     7, 29
        vmr     8, 30
        bl      do_mul          # r^2 r^1
        xxpermdi 58, 58, 36, 0x3                # r0
        xxpermdi 59, 59, 37, 0x3                # r1
        xxpermdi 60, 60, 38, 0x3                # r2
        xxpermdi 61, 61, 39, 0x3                # r3
        xxpermdi 62, 62, 40, 0x3                # r4
        xxpermdi 36, 36, 36, 0x3
        xxpermdi 37, 37, 37, 0x3
        xxpermdi 38, 38, 38, 0x3
        xxpermdi 39, 39, 39, 0x3
        xxpermdi 40, 40, 40, 0x3
        vspltisb 13, 2
        vsld    9, 27, 13
        vsld    10, 28, 13
        vsld    11, 29, 13
        vsld    12, 30, 13
        vaddudm 0, 9, 27
        vaddudm 1, 10, 28
        vaddudm 2, 11, 29
        vaddudm 3, 12, 30

        bl      do_mul          # r^4 r^3
        vmrgow  26, 26, 4
        vmrgow  27, 27, 5
        vmrgow  28, 28, 6
        vmrgow  29, 29, 7
        vmrgow  30, 30, 8
        vspltisb 13, 2
        vsld    9, 27, 13
        vsld    10, 28, 13
        vsld    11, 29, 13
        vsld    12, 30, 13
        vaddudm 0, 9, 27
        vaddudm 1, 10, 28
        vaddudm 2, 11, 29
        vaddudm 3, 12, 30

        # r^2 r^4
        xxlor   0, 58, 58
        xxlor   1, 59, 59
        xxlor   2, 60, 60
        xxlor   3, 61, 61
        xxlor   4, 62, 62
        xxlor   5, 32, 32
        xxlor   6, 33, 33
        xxlor   7, 34, 34
        xxlor   8, 35, 35

        vspltw  9, 26, 3
        vspltw  10, 26, 2
        vmrgow  26, 10, 9
        vspltw  9, 27, 3
        vspltw  10, 27, 2
        vmrgow  27, 10, 9
        vspltw  9, 28, 3
        vspltw  10, 28, 2
        vmrgow  28, 10, 9
        vspltw  9, 29, 3
        vspltw  10, 29, 2
        vmrgow  29, 10, 9
        vspltw  9, 30, 3
        vspltw  10, 30, 2
        vmrgow  30, 10, 9

        vsld    9, 27, 13
        vsld    10, 28, 13
        vsld    11, 29, 13
        vsld    12, 30, 13
        vaddudm 0, 9, 27
        vaddudm 1, 10, 28
        vaddudm 2, 11, 29
        vaddudm 3, 12, 30
.endm

SYM_FUNC_START_LOCAL(do_mul)
        mul_odd

        # do reduction ( h %= p )
        # carry reduction
        vspltisb 9, 2
        vsrd    10, 14, 31
        vsrd    11, 17, 31
        vand    7, 17, 25
        vand    4, 14, 25
        vaddudm 18, 18, 11
        vsrd    12, 18, 31
        vaddudm 15, 15, 10

        vsrd    11, 15, 31
        vand    8, 18, 25
        vand    5, 15, 25
        vaddudm 4, 4, 12
        vsld    10, 12, 9
        vaddudm 6, 16, 11

        vsrd    13, 6, 31
        vand    6, 6, 25
        vaddudm 4, 4, 10
        vsrd    10, 4, 31
        vaddudm 7, 7, 13

        vsrd    11, 7, 31
        vand    7, 7, 25
        vand    4, 4, 25
        vaddudm 5, 5, 10
        vaddudm 8, 8, 11
        blr
SYM_FUNC_END(do_mul)

#
# init key
#
.macro do_poly1305_init
        addis   10, 2, rmask@toc@ha
        addi    10, 10, rmask@toc@l

        ld      11, 0(10)
        ld      12, 8(10)

        li      14, 16
        li      15, 32
        addis   10, 2, cnum@toc@ha
        addi    10, 10, cnum@toc@l
        lvx     25, 0, 10       # v25 - mask
        lvx     31, 14, 10      # v31 = 1a
        lvx     19, 15, 10      # v19 = 1 << 24
        lxv     24, 48(10)      # vs24
        lxv     25, 64(10)      # vs25

        # initialize
        # load key from r3 to vectors
        ld      9, 24(3)
        ld      10, 32(3)
        and.    9, 9, 11
        and.    10, 10, 12

        # break 26 bits
        extrdi  14, 9, 26, 38
        extrdi  15, 9, 26, 12
        extrdi  16, 9, 12, 0
        mtvsrdd 58, 0, 14
        insrdi  16, 10, 14, 38
        mtvsrdd 59, 0, 15
        extrdi  17, 10, 26, 24
        mtvsrdd 60, 0, 16
        extrdi  18, 10, 24, 0
        mtvsrdd 61, 0, 17
        mtvsrdd 62, 0, 18

        # r1 = r1 * 5, r2 = r2 * 5, r3 = r3 * 5, r4 = r4 * 5
        li      9, 5
        mtvsrdd 36, 0, 9
        vmulouw 0, 27, 4                # v0 = rr0
        vmulouw 1, 28, 4                # v1 = rr1
        vmulouw 2, 29, 4                # v2 = rr2
        vmulouw 3, 30, 4                # v3 = rr3
.endm

#
# poly1305_p10le_4blocks( uint8_t *k, uint32_t mlen, uint8_t *m)
#  k = 32 bytes key
#  r3 = k (r, s)
#  r4 = mlen
#  r5 = m
#
SYM_FUNC_START(poly1305_p10le_4blocks)
.align 5
        cmpdi   5, 64
        blt     Out_no_poly1305

        SAVE_REGS

        do_poly1305_init

        li      21, 0   # counter to message

        poly1305_setup_r

        # load previous H state
        # break/convert r6 to 26 bits
        ld      9, 0(3)
        ld      10, 8(3)
        ld      19, 16(3)
        sldi    19, 19, 24
        mtvsrdd 41, 0, 19
        extrdi  14, 9, 26, 38
        extrdi  15, 9, 26, 12
        extrdi  16, 9, 12, 0
        mtvsrdd 36, 0, 14
        insrdi  16, 10, 14, 38
        mtvsrdd 37, 0, 15
        extrdi  17, 10, 26, 24
        mtvsrdd 38, 0, 16
        extrdi  18, 10, 24, 0
        mtvsrdd 39, 0, 17
        mtvsrdd 40, 0, 18
        vor     8, 8, 9

        # input m1 m2
        add     20, 4, 21
        xxlor   49, 24, 24
        xxlor   50, 25, 25
        lxvw4x  43, 0, 20
        addi    17, 20, 16
        lxvw4x  44, 0, 17
        vperm   14, 11, 12, 17
        vperm   15, 11, 12, 18
        vand    9, 14, 25       # a0
        vsrd    10, 14, 31      # >> 26
        vsrd    11, 10, 31      # 12 bits left
        vand    10, 10, 25      # a1
        vspltisb 13, 12
        vand    16, 15, 25
        vsld    12, 16, 13
        vor     11, 11, 12
        vand    11, 11, 25      # a2
        vspltisb 13, 14
        vsrd    12, 15, 13      # >> 14
        vsrd    13, 12, 31      # >> 26, a4
        vand    12, 12, 25      # a3

        vaddudm 20, 4, 9
        vaddudm 21, 5, 10
        vaddudm 22, 6, 11
        vaddudm 23, 7, 12
        vaddudm 24, 8, 13

        # m3 m4
        addi    17, 17, 16
        lxvw4x  43, 0, 17
        addi    17, 17, 16
        lxvw4x  44, 0, 17
        vperm   14, 11, 12, 17
        vperm   15, 11, 12, 18
        vand    9, 14, 25       # a0
        vsrd    10, 14, 31      # >> 26
        vsrd    11, 10, 31      # 12 bits left
        vand    10, 10, 25      # a1
        vspltisb 13, 12
        vand    16, 15, 25
        vsld    12, 16, 13
        vspltisb 13, 14
        vor     11, 11, 12
        vand    11, 11, 25      # a2
        vsrd    12, 15, 13      # >> 14
        vsrd    13, 12, 31      # >> 26, a4
        vand    12, 12, 25      # a3

        # Smash 4 message blocks into 5 vectors of [m4,  m2,  m3,  m1]
        vmrgow  4, 9, 20
        vmrgow  5, 10, 21
        vmrgow  6, 11, 22
        vmrgow  7, 12, 23
        vmrgow  8, 13, 24
        vaddudm 8, 8, 19

        addi    5, 5, -64       # len -= 64
        addi    21, 21, 64      # offset += 64

        li      9, 64
        divdu   31, 5, 9

        cmpdi   31, 0
        ble     Skip_block_loop

        mtctr   31

# h4 =   m1 * r⁴ + m2 * r³ + m3 * r² + m4 * r
# Rewrite the polynominal sum of product as follows,
# h1 = (h0 + m1) * r^2, h2 = (h0 + m2) * r^2
# h3 = (h1 + m3) * r^2, h4 = (h2 + m4) * r^2  --> (h0 + m1) r*4 + (h3 + m3) r^2, (h0 + m2) r^4 + (h0 + m4) r^2
#  .... Repeat
# h5 = (h3 + m5) * r^2, h6 = (h4 + m6) * r^2  -->
# h7 = (h5 + m7) * r^2, h8 = (h6 + m8) * r^1  --> m5 * r^4 + m6 * r^3 + m7 * r^2 + m8 * r
#
loop_4blocks:

        # Multiply odd words and even words
        mul_odd
        mul_even
        # carry reduction
        vspltisb 9, 2
        vsrd    10, 14, 31
        vsrd    11, 17, 31
        vand    7, 17, 25
        vand    4, 14, 25
        vaddudm 18, 18, 11
        vsrd    12, 18, 31
        vaddudm 15, 15, 10

        vsrd    11, 15, 31
        vand    8, 18, 25
        vand    5, 15, 25
        vaddudm 4, 4, 12
        vsld    10, 12, 9
        vaddudm 6, 16, 11

        vsrd    13, 6, 31
        vand    6, 6, 25
        vaddudm 4, 4, 10
        vsrd    10, 4, 31
        vaddudm 7, 7, 13

        vsrd    11, 7, 31
        vand    7, 7, 25
        vand    4, 4, 25
        vaddudm 5, 5, 10
        vaddudm 8, 8, 11

        # input m1  m2  m3  m4
        add     20, 4, 21
        xxlor   49, 24, 24
        xxlor   50, 25, 25
        lxvw4x  43, 0, 20
        addi    17, 20, 16
        lxvw4x  44, 0, 17
        vperm   14, 11, 12, 17
        vperm   15, 11, 12, 18
        addi    17, 17, 16
        lxvw4x  43, 0, 17
        addi    17, 17, 16
        lxvw4x  44, 0, 17
        vperm   17, 11, 12, 17
        vperm   18, 11, 12, 18

        vand    20, 14, 25      # a0
        vand    9, 17, 25       # a0
        vsrd    21, 14, 31      # >> 26
        vsrd    22, 21, 31      # 12 bits left
        vsrd    10, 17, 31      # >> 26
        vsrd    11, 10, 31      # 12 bits left

        vand    21, 21, 25      # a1
        vand    10, 10, 25      # a1

        vspltisb 13, 12
        vand    16, 15, 25
        vsld    23, 16, 13
        vor     22, 22, 23
        vand    22, 22, 25      # a2
        vand    16, 18, 25
        vsld    12, 16, 13
        vor     11, 11, 12
        vand    11, 11, 25      # a2
        vspltisb 13, 14
        vsrd    23, 15, 13      # >> 14
        vsrd    24, 23, 31      # >> 26, a4
        vand    23, 23, 25      # a3
        vsrd    12, 18, 13      # >> 14
        vsrd    13, 12, 31      # >> 26, a4
        vand    12, 12, 25      # a3

        vaddudm 4, 4, 20
        vaddudm 5, 5, 21
        vaddudm 6, 6, 22
        vaddudm 7, 7, 23
        vaddudm 8, 8, 24

        # Smash 4 message blocks into 5 vectors of [m4,  m2,  m3,  m1]
        vmrgow  4, 9, 4
        vmrgow  5, 10, 5
        vmrgow  6, 11, 6
        vmrgow  7, 12, 7
        vmrgow  8, 13, 8
        vaddudm 8, 8, 19

        addi    5, 5, -64       # len -= 64
        addi    21, 21, 64      # offset += 64

        bdnz    loop_4blocks

Skip_block_loop:
        xxlor   58, 0, 0
        xxlor   59, 1, 1
        xxlor   60, 2, 2
        xxlor   61, 3, 3
        xxlor   62, 4, 4
        xxlor   32, 5, 5
        xxlor   33, 6, 6
        xxlor   34, 7, 7
        xxlor   35, 8, 8

        # Multiply odd words and even words
        mul_odd
        mul_even

        # Sum the products.
        xxpermdi 41, 31, 46, 0
        xxpermdi 42, 31, 47, 0
        vaddudm 4, 14, 9
        xxpermdi 36, 31, 36, 3
        vaddudm 5, 15, 10
        xxpermdi 37, 31, 37, 3
        xxpermdi 43, 31, 48, 0
        vaddudm 6, 16, 11
        xxpermdi 38, 31, 38, 3
        xxpermdi 44, 31, 49, 0
        vaddudm 7, 17, 12
        xxpermdi 39, 31, 39, 3
        xxpermdi 45, 31, 50, 0
        vaddudm 8, 18, 13
        xxpermdi 40, 31, 40, 3

        # carry reduction
        vspltisb 9, 2
        vsrd    10, 4, 31
        vsrd    11, 7, 31
        vand    7, 7, 25
        vand    4, 4, 25
        vaddudm 8, 8, 11
        vsrd    12, 8, 31
        vaddudm 5, 5, 10

        vsrd    11, 5, 31
        vand    8, 8, 25
        vand    5, 5, 25
        vaddudm 4, 4, 12
        vsld    10, 12, 9
        vaddudm 6, 6, 11

        vsrd    13, 6, 31
        vand    6, 6, 25
        vaddudm 4, 4, 10
        vsrd    10, 4, 31
        vaddudm 7, 7, 13

        vsrd    11, 7, 31
        vand    7, 7, 25
        vand    4, 4, 25
        vaddudm 5, 5, 10
        vsrd    10, 5, 31
        vand    5, 5, 25
        vaddudm 6, 6, 10
        vaddudm 8, 8, 11

        b       do_final_update

do_final_update:
        # combine 26 bit limbs
        # v4, v5, v6, v7 and v8 are 26 bit vectors
        vsld    5, 5, 31
        vor     20, 4, 5
        vspltisb 11, 12
        vsrd    12, 6, 11
        vsld    6, 6, 31
        vsld    6, 6, 31
        vor     20, 20, 6
        vspltisb 11, 14
        vsld    7, 7, 11
        vor     21, 7, 12
        mfvsrld 16, 40          # save last 2 bytes
        vsld    8, 8, 11
        vsld    8, 8, 31
        vor     21, 21, 8
        mfvsrld 17, 52
        mfvsrld 19, 53
        srdi    16, 16, 24

        std     17, 0(3)
        std     19, 8(3)
        stw     16, 16(3)

Out_loop:
        li      3, 0

        RESTORE_REGS

        blr

Out_no_poly1305:
        li      3, 0
        blr
SYM_FUNC_END(poly1305_p10le_4blocks)

#
# =======================================================================
# The following functions implement 64 x 64 bits multiplication poly1305.
#
SYM_FUNC_START_LOCAL(Poly1305_init_64)
        #  mask 0x0FFFFFFC0FFFFFFC
        #  mask 0x0FFFFFFC0FFFFFFF
        addis   10, 2, rmask@toc@ha
        addi    10, 10, rmask@toc@l
        ld      11, 0(10)
        ld      12, 8(10)

        # initialize
        # load key from r3
        ld      9, 24(3)
        ld      10, 32(3)
        and.    9, 9, 11        # cramp mask r0
        and.    10, 10, 12      # cramp mask r1

        srdi    21, 10, 2
        add     19, 21, 10      # s1: r19 - (r1 >> 2) *5

        # setup r and s
        li      25, 0
        mtvsrdd 32+0, 9, 19     # r0, s1
        mtvsrdd 32+1, 10, 9     # r1, r0
        mtvsrdd 32+2, 19, 25    # s1
        mtvsrdd 32+3, 9, 25     # r0

        blr
SYM_FUNC_END(Poly1305_init_64)

# Poly1305_mult
# v6 = (h0, h1), v8 = h2
# v0 = (r0, s1), v1 = (r1, r0), v2 = s1, v3 = r0
#
# Output: v7, v10, v11
#
SYM_FUNC_START_LOCAL(Poly1305_mult)
        #
        #       d0 = h0 * r0 + h1 * s1
        vmsumudm        7, 6, 0, 9              # h0 * r0, h1 * s1

        #       d1 = h0 * r1 + h1 * r0 + h2 * s1
        vmsumudm        11, 6, 1, 9             # h0 * r1, h1 * r0
        vmsumudm        10, 8, 2, 11            # d1 += h2 * s1

        #       d2 = r0
        vmsumudm        11, 8, 3, 9             # d2 = h2 * r0
        blr
SYM_FUNC_END(Poly1305_mult)

#
# carry reduction
# h %=p
#
# Input: v7, v10, v11
# Output: r27, r28, r29
#
SYM_FUNC_START_LOCAL(Carry_reduction)
        mfvsrld 27, 32+7
        mfvsrld 28, 32+10
        mfvsrld 29, 32+11
        mfvsrd  20, 32+7        # h0.h
        mfvsrd  21, 32+10       # h1.h

        addc    28, 28, 20
        adde    29, 29, 21
        srdi    22, 29, 0x2
        sldi    23, 22, 0x2
        add     23, 23, 22      # (h2 & 3) * 5
        addc    27, 27, 23      # h0
        addze   28, 28          # h1
        andi.   29, 29, 0x3     # h2
        blr
SYM_FUNC_END(Carry_reduction)

#
# poly1305 multiplication
# h *= r, h %= p
#       d0 = h0 * r0 + h1 * s1
#       d1 = h0 * r1 + h1 * r0 + h2 * s1
#       d2 = h0 * r0
#
#
# unsigned int poly1305_test_64s(unisgned char *state, const byte *src, size_t len, highbit)
#   - no highbit if final leftover block (highbit = 0)
#
SYM_FUNC_START(poly1305_64s)
        cmpdi   5, 0
        ble     Out_no_poly1305_64

        mflr 0
        std 0, 16(1)
        stdu 1,-400(1)

        SAVE_GPR 14, 112, 1
        SAVE_GPR 15, 120, 1
        SAVE_GPR 16, 128, 1
        SAVE_GPR 17, 136, 1
        SAVE_GPR 18, 144, 1
        SAVE_GPR 19, 152, 1
        SAVE_GPR 20, 160, 1
        SAVE_GPR 21, 168, 1
        SAVE_GPR 22, 176, 1
        SAVE_GPR 23, 184, 1
        SAVE_GPR 24, 192, 1
        SAVE_GPR 25, 200, 1
        SAVE_GPR 26, 208, 1
        SAVE_GPR 27, 216, 1
        SAVE_GPR 28, 224, 1
        SAVE_GPR 29, 232, 1
        SAVE_GPR 30, 240, 1
        SAVE_GPR 31, 248, 1

        # Init poly1305
        bl Poly1305_init_64

        li 25, 0                        # offset to inp and outp

        add 11, 25, 4

        # load h
        # h0, h1, h2?
        ld      27, 0(3)
        ld      28, 8(3)
        lwz     29, 16(3)

        li      30, 16
        divdu   31, 5, 30

        mtctr   31

        mr      24, 6           # highbit

Loop_block_64:
        vxor    9, 9, 9

        ld      20, 0(11)
        ld      21, 8(11)
        addi    11, 11, 16

        addc    27, 27, 20
        adde    28, 28, 21
        adde    29, 29, 24

        li      22, 0
        mtvsrdd 32+6, 27, 28    # h0, h1
        mtvsrdd 32+8, 29, 22    # h2

        bl      Poly1305_mult

        bl      Carry_reduction

        bdnz    Loop_block_64

        std     27, 0(3)
        std     28, 8(3)
        stw     29, 16(3)

        li      3, 0

        RESTORE_GPR 14, 112, 1
        RESTORE_GPR 15, 120, 1
        RESTORE_GPR 16, 128, 1
        RESTORE_GPR 17, 136, 1
        RESTORE_GPR 18, 144, 1
        RESTORE_GPR 19, 152, 1
        RESTORE_GPR 20, 160, 1
        RESTORE_GPR 21, 168, 1
        RESTORE_GPR 22, 176, 1
        RESTORE_GPR 23, 184, 1
        RESTORE_GPR 24, 192, 1
        RESTORE_GPR 25, 200, 1
        RESTORE_GPR 26, 208, 1
        RESTORE_GPR 27, 216, 1
        RESTORE_GPR 28, 224, 1
        RESTORE_GPR 29, 232, 1
        RESTORE_GPR 30, 240, 1
        RESTORE_GPR 31, 248, 1

        addi    1, 1, 400
        ld 0, 16(1)
        mtlr 0

        blr

Out_no_poly1305_64:
        li      3, 0
        blr
SYM_FUNC_END(poly1305_64s)

#
# Input: r3 = h, r4 = s, r5 = mac
# mac = h + s
#
SYM_FUNC_START(poly1305_emit_64)
        ld      10, 0(3)
        ld      11, 8(3)
        ld      12, 16(3)

        # compare modulus
        # h + 5 + (-p)
        mr      6, 10
        mr      7, 11
        mr      8, 12
        addic.  6, 6, 5
        addze   7, 7
        addze   8, 8
        srdi    9, 8, 2         # overflow?
        cmpdi   9, 0
        beq     Skip_h64
        mr      10, 6
        mr      11, 7
        mr      12, 8

Skip_h64:
        ld      6, 0(4)
        ld      7, 8(4)
        addc    10, 10, 6
        adde    11, 11, 7
        addze   12, 12

        std     10, 0(5)
        std     11, 8(5)
        blr
SYM_FUNC_END(poly1305_emit_64)

SYM_DATA_START_LOCAL(RMASK)
.align 5
rmask:
.byte   0xff, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f, 0xfc, 0xff, 0xff, 0x0f
cnum:
.long   0x03ffffff, 0x00000000, 0x03ffffff, 0x00000000
.long   0x1a, 0x00, 0x1a, 0x00
.long   0x01000000, 0x01000000, 0x01000000, 0x01000000
.long   0x00010203, 0x04050607, 0x10111213, 0x14151617
.long   0x08090a0b, 0x0c0d0e0f, 0x18191a1b, 0x1c1d1e1f
SYM_DATA_END(RMASK)