/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * Fast SHA-256 implementation for SPE instruction set (PPC)
 *
 * This code makes use of the SPE SIMD instruction set as defined in
 * http://cache.freescale.com/files/32bit/doc/ref_manual/SPEPIM.pdf
 * Implementation is based on optimization guide notes from
 * http://cache.freescale.com/files/32bit/doc/app_note/AN2665.pdf
 *
 * Copyright (c) 2015 Markus Stockhausen <stockhausen@collogia.de>
 */

#include <asm/ppc_asm.h>
#include <asm/asm-offsets.h>

#define rHP     r3      /* pointer to hash values in memory             */
#define rKP     r24     /* pointer to round constants                   */
#define rWP     r4      /* pointer to input data                        */

#define rH0     r5      /* 8 32 bit hash values in 8 registers          */
#define rH1     r6
#define rH2     r7
#define rH3     r8
#define rH4     r9
#define rH5     r10
#define rH6     r11
#define rH7     r12

#define rW0     r14     /* 64 bit registers. 16 words in 8 registers    */
#define rW1     r15
#define rW2     r16
#define rW3     r17
#define rW4     r18
#define rW5     r19
#define rW6     r20
#define rW7     r21

#define rT0     r22     /* 64 bit temporaries                           */
#define rT1     r23
#define rT2     r0      /* 32 bit temporaries                           */
#define rT3     r25

#define CMP_KN_LOOP
#define CMP_KC_LOOP \
        cmpwi           rT1,0;

#define INITIALIZE \
        stwu            r1,-128(r1);    /* create stack frame           */ \
        evstdw          r14,8(r1);      /* We must save non volatile    */ \
        evstdw          r15,16(r1);     /* registers. Take the chance   */ \
        evstdw          r16,24(r1);     /* and save the SPE part too    */ \
        evstdw          r17,32(r1);                                        \
        evstdw          r18,40(r1);                                        \
        evstdw          r19,48(r1);                                        \
        evstdw          r20,56(r1);                                        \
        evstdw          r21,64(r1);                                        \
        evstdw          r22,72(r1);                                        \
        evstdw          r23,80(r1);                                        \
        stw             r24,88(r1);     /* save normal registers        */ \
        stw             r25,92(r1);


#define FINALIZE \
        evldw           r14,8(r1);      /* restore SPE registers        */ \
        evldw           r15,16(r1);                                        \
        evldw           r16,24(r1);                                        \
        evldw           r17,32(r1);                                        \
        evldw           r18,40(r1);                                        \
        evldw           r19,48(r1);                                        \
        evldw           r20,56(r1);                                        \
        evldw           r21,64(r1);                                        \
        evldw           r22,72(r1);                                        \
        evldw           r23,80(r1);                                        \
        lwz             r24,88(r1);     /* restore normal registers     */ \
        lwz             r25,92(r1);                                        \
        xor             r0,r0,r0;                                          \
        stw             r0,8(r1);       /* Delete sensitive data        */ \
        stw             r0,16(r1);      /* that we might have pushed    */ \
        stw             r0,24(r1);      /* from other context that runs */ \
        stw             r0,32(r1);      /* the same code. Assume that   */ \
        stw             r0,40(r1);      /* the lower part of the GPRs   */ \
        stw             r0,48(r1);      /* was already overwritten on   */ \
        stw             r0,56(r1);      /* the way down to here         */ \
        stw             r0,64(r1);                                         \
        stw             r0,72(r1);                                         \
        stw             r0,80(r1);                                         \
        addi            r1,r1,128;      /* cleanup stack frame          */

#ifdef __BIG_ENDIAN__
#define LOAD_DATA(reg, off) \
        lwz             reg,off(rWP);   /* load data                    */
#define NEXT_BLOCK \
        addi            rWP,rWP,64;     /* increment per block          */
#else
#define LOAD_DATA(reg, off) \
        lwbrx           reg,0,rWP;      /* load data                    */ \
        addi            rWP,rWP,4;      /* increment per word           */
#define NEXT_BLOCK                      /* nothing to do                */
#endif

#define R_LOAD_W(a, b, c, d, e, f, g, h, w, off) \
        LOAD_DATA(w, off)               /* 1: W                         */ \
        rotrwi          rT0,e,6;        /* 1: S1 = e rotr 6             */ \
        rotrwi          rT1,e,11;       /* 1: S1' = e rotr 11           */ \
        rotrwi          rT2,e,25;       /* 1: S1" = e rotr 25           */ \
        xor             rT0,rT0,rT1;    /* 1: S1 = S1 xor S1'           */ \
        and             rT3,e,f;        /* 1: ch = e and f              */ \
        xor             rT0,rT0,rT2;    /* 1: S1 = S1 xor S1"           */ \
        andc            rT1,g,e;        /* 1: ch' = ~e and g            */ \
        lwz             rT2,off(rKP);   /* 1: K                         */ \
        xor             rT3,rT3,rT1;    /* 1: ch = ch xor ch'           */ \
        add             h,h,rT0;        /* 1: temp1 = h + S1            */ \
        add             rT3,rT3,w;      /* 1: temp1' = ch + w           */ \
        rotrwi          rT0,a,2;        /* 1: S0 = a rotr 2             */ \
        add             h,h,rT3;        /* 1: temp1 = temp1 + temp1'    */ \
        rotrwi          rT1,a,13;       /* 1: S0' = a rotr 13           */ \
        add             h,h,rT2;        /* 1: temp1 = temp1 + K         */ \
        rotrwi          rT3,a,22;       /* 1: S0" = a rotr 22           */ \
        xor             rT0,rT0,rT1;    /* 1: S0 = S0 xor S0'           */ \
        add             d,d,h;          /* 1: d = d + temp1             */ \
        xor             rT3,rT0,rT3;    /* 1: S0 = S0 xor S0"           */ \
        evmergelo       w,w,w;          /*    shift W                   */ \
        or              rT2,a,b;        /* 1: maj = a or b              */ \
        and             rT1,a,b;        /* 1: maj' = a and b            */ \
        and             rT2,rT2,c;      /* 1: maj = maj and c           */ \
        LOAD_DATA(w, off+4)             /* 2: W                         */ \
        or              rT2,rT1,rT2;    /* 1: maj = maj or maj'         */ \
        rotrwi          rT0,d,6;        /* 2: S1 = e rotr 6             */ \
        add             rT3,rT3,rT2;    /* 1: temp2 = S0 + maj          */ \
        rotrwi          rT1,d,11;       /* 2: S1' = e rotr 11           */ \
        add             h,h,rT3;        /* 1: h = temp1 + temp2         */ \
        rotrwi          rT2,d,25;       /* 2: S1" = e rotr 25           */ \
        xor             rT0,rT0,rT1;    /* 2: S1 = S1 xor S1'           */ \
        and             rT3,d,e;        /* 2: ch = e and f              */ \
        xor             rT0,rT0,rT2;    /* 2: S1 = S1 xor S1"           */ \
        andc            rT1,f,d;        /* 2: ch' = ~e and g            */ \
        lwz             rT2,off+4(rKP); /* 2: K                         */ \
        xor             rT3,rT3,rT1;    /* 2: ch = ch xor ch'           */ \
        add             g,g,rT0;        /* 2: temp1 = h + S1            */ \
        add             rT3,rT3,w;      /* 2: temp1' = ch + w           */ \
        rotrwi          rT0,h,2;        /* 2: S0 = a rotr 2             */ \
        add             g,g,rT3;        /* 2: temp1 = temp1 + temp1'    */ \
        rotrwi          rT1,h,13;       /* 2: S0' = a rotr 13           */ \
        add             g,g,rT2;        /* 2: temp1 = temp1 + K         */ \
        rotrwi          rT3,h,22;       /* 2: S0" = a rotr 22           */ \
        xor             rT0,rT0,rT1;    /* 2: S0 = S0 xor S0'           */ \
        or              rT2,h,a;        /* 2: maj = a or b              */ \
        xor             rT3,rT0,rT3;    /* 2: S0 = S0 xor S0"           */ \
        and             rT1,h,a;        /* 2: maj' = a and b            */ \
        and             rT2,rT2,b;      /* 2: maj = maj and c           */ \
        add             c,c,g;          /* 2: d = d + temp1             */ \
        or              rT2,rT1,rT2;    /* 2: maj = maj or maj'         */ \
        add             rT3,rT3,rT2;    /* 2: temp2 = S0 + maj          */ \
        add             g,g,rT3         /* 2: h = temp1 + temp2         */

#define R_CALC_W(a, b, c, d, e, f, g, h, w0, w1, w4, w5, w7, k, off) \
        rotrwi          rT2,e,6;        /* 1: S1 = e rotr 6             */ \
        evmergelohi     rT0,w0,w1;      /*    w[-15]                    */ \
        rotrwi          rT3,e,11;       /* 1: S1' = e rotr 11           */ \
        evsrwiu         rT1,rT0,3;      /*    s0 = w[-15] >> 3          */ \
        xor             rT2,rT2,rT3;    /* 1: S1 = S1 xor S1'           */ \
        evrlwi          rT0,rT0,25;     /*    s0' = w[-15] rotr 7       */ \
        rotrwi          rT3,e,25;       /* 1: S1' = e rotr 25           */ \
        evxor           rT1,rT1,rT0;    /*    s0 = s0 xor s0'           */ \
        xor             rT2,rT2,rT3;    /* 1: S1 = S1 xor S1'           */ \
        evrlwi          rT0,rT0,21;     /*    s0' = w[-15] rotr 18      */ \
        add             h,h,rT2;        /* 1: temp1 = h + S1            */ \
        evxor           rT0,rT0,rT1;    /*    s0 = s0 xor s0'           */ \
        and             rT2,e,f;        /* 1: ch = e and f              */ \
        evaddw          w0,w0,rT0;      /*    w = w[-16] + s0           */ \
        andc            rT3,g,e;        /* 1: ch' = ~e and g            */ \
        evsrwiu         rT0,w7,10;      /*    s1 = w[-2] >> 10          */ \
        xor             rT2,rT2,rT3;    /* 1: ch = ch xor ch'           */ \
        evrlwi          rT1,w7,15;      /*    s1' = w[-2] rotr 17       */ \
        add             h,h,rT2;        /* 1: temp1 = temp1 + ch        */ \
        evxor           rT0,rT0,rT1;    /*    s1 = s1 xor s1'           */ \
        rotrwi          rT2,a,2;        /* 1: S0 = a rotr 2             */ \
        evrlwi          rT1,w7,13;      /*    s1' = w[-2] rotr 19       */ \
        rotrwi          rT3,a,13;       /* 1: S0' = a rotr 13           */ \
        evxor           rT0,rT0,rT1;    /*    s1 = s1 xor s1'           */ \
        xor             rT2,rT2,rT3;    /* 1: S0 = S0 xor S0'           */ \
        evldw           rT1,off(rKP);   /*    k                         */ \
        rotrwi          rT3,a,22;       /* 1: S0' = a rotr 22           */ \
        evaddw          w0,w0,rT0;      /*    w = w + s1                */ \
        xor             rT2,rT2,rT3;    /* 1: S0 = S0 xor S0'           */ \
        evmergelohi     rT0,w4,w5;      /*    w[-7]                     */ \
        and             rT3,a,b;        /* 1: maj = a and b             */ \
        evaddw          w0,w0,rT0;      /*    w = w + w[-7]             */ \
        CMP_K##k##_LOOP                                                    \
        add             rT2,rT2,rT3;    /* 1: temp2 = S0 + maj          */ \
        evaddw          rT1,rT1,w0;     /*    wk = w + k                */ \
        xor             rT3,a,b;        /* 1: maj = a xor b             */ \
        evmergehi       rT0,rT1,rT1;    /*    wk1/wk2                   */ \
        and             rT3,rT3,c;      /* 1: maj = maj and c           */ \
        add             h,h,rT0;        /* 1: temp1 = temp1 + wk        */ \
        add             rT2,rT2,rT3;    /* 1: temp2 = temp2 + maj       */ \
        add             g,g,rT1;        /* 2: temp1 = temp1 + wk        */ \
        add             d,d,h;          /* 1: d = d + temp1             */ \
        rotrwi          rT0,d,6;        /* 2: S1 = e rotr 6             */ \
        add             h,h,rT2;        /* 1: h = temp1 + temp2         */ \
        rotrwi          rT1,d,11;       /* 2: S1' = e rotr 11           */ \
        rotrwi          rT2,d,25;       /* 2: S" = e rotr 25            */ \
        xor             rT0,rT0,rT1;    /* 2: S1 = S1 xor S1'           */ \
        and             rT3,d,e;        /* 2: ch = e and f              */ \
        xor             rT0,rT0,rT2;    /* 2: S1 = S1 xor S1"           */ \
        andc            rT1,f,d;        /* 2: ch' = ~e and g            */ \
        add             g,g,rT0;        /* 2: temp1 = h + S1            */ \
        xor             rT3,rT3,rT1;    /* 2: ch = ch xor ch'           */ \
        rotrwi          rT0,h,2;        /* 2: S0 = a rotr 2             */ \
        add             g,g,rT3;        /* 2: temp1 = temp1 + ch        */ \
        rotrwi          rT1,h,13;       /* 2: S0' = a rotr 13           */ \
        rotrwi          rT3,h,22;       /* 2: S0" = a rotr 22           */ \
        xor             rT0,rT0,rT1;    /* 2: S0 = S0 xor S0'           */ \
        or              rT2,h,a;        /* 2: maj = a or b              */ \
        and             rT1,h,a;        /* 2: maj' = a and b            */ \
        and             rT2,rT2,b;      /* 2: maj = maj and c           */ \
        xor             rT3,rT0,rT3;    /* 2: S0 = S0 xor S0"           */ \
        or              rT2,rT1,rT2;    /* 2: maj = maj or maj'         */ \
        add             c,c,g;          /* 2: d = d + temp1             */ \
        add             rT3,rT3,rT2;    /* 2: temp2 = S0 + maj          */ \
        add             g,g,rT3         /* 2: h = temp1 + temp2         */

_GLOBAL(ppc_spe_sha256_transform)
        INITIALIZE

        mtctr           r5
        lwz             rH0,0(rHP)
        lwz             rH1,4(rHP)
        lwz             rH2,8(rHP)
        lwz             rH3,12(rHP)
        lwz             rH4,16(rHP)
        lwz             rH5,20(rHP)
        lwz             rH6,24(rHP)
        lwz             rH7,28(rHP)

ppc_spe_sha256_main:
        lis             rKP,PPC_SPE_SHA256_K@ha
        addi            rKP,rKP,PPC_SPE_SHA256_K@l

        R_LOAD_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, rW0, 0)
        R_LOAD_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, rW1, 8)
        R_LOAD_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, rW2, 16)
        R_LOAD_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, rW3, 24)
        R_LOAD_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7, rW4, 32)
        R_LOAD_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5, rW5, 40)
        R_LOAD_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3, rW6, 48)
        R_LOAD_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1, rW7, 56)
ppc_spe_sha256_16_rounds:
        addi            rKP,rKP,64
        R_CALC_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7,
                 rW0, rW1, rW4, rW5, rW7, N, 0)
        R_CALC_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5,
                 rW1, rW2, rW5, rW6, rW0, N, 8)
        R_CALC_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3,
                 rW2, rW3, rW6, rW7, rW1, N, 16)
        R_CALC_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1,
                 rW3, rW4, rW7, rW0, rW2, N, 24)
        R_CALC_W(rH0, rH1, rH2, rH3, rH4, rH5, rH6, rH7,
                 rW4, rW5, rW0, rW1, rW3, N, 32)
        R_CALC_W(rH6, rH7, rH0, rH1, rH2, rH3, rH4, rH5,
                 rW5, rW6, rW1, rW2, rW4, N, 40)
        R_CALC_W(rH4, rH5, rH6, rH7, rH0, rH1, rH2, rH3,
                 rW6, rW7, rW2, rW3, rW5, N, 48)
        R_CALC_W(rH2, rH3, rH4, rH5, rH6, rH7, rH0, rH1,
                 rW7, rW0, rW3, rW4, rW6, C, 56)
        bt              gt,ppc_spe_sha256_16_rounds

        lwz             rW0,0(rHP)
        NEXT_BLOCK
        lwz             rW1,4(rHP)
        lwz             rW2,8(rHP)
        lwz             rW3,12(rHP)
        lwz             rW4,16(rHP)
        lwz             rW5,20(rHP)
        lwz             rW6,24(rHP)
        lwz             rW7,28(rHP)

        add             rH0,rH0,rW0
        stw             rH0,0(rHP)
        add             rH1,rH1,rW1
        stw             rH1,4(rHP)
        add             rH2,rH2,rW2
        stw             rH2,8(rHP)
        add             rH3,rH3,rW3
        stw             rH3,12(rHP)
        add             rH4,rH4,rW4
        stw             rH4,16(rHP)
        add             rH5,rH5,rW5
        stw             rH5,20(rHP)
        add             rH6,rH6,rW6
        stw             rH6,24(rHP)
        add             rH7,rH7,rW7
        stw             rH7,28(rHP)

        bdnz            ppc_spe_sha256_main

        FINALIZE
        blr

.data
.align 5
PPC_SPE_SHA256_K:
        .long 0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5
        .long 0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5
        .long 0xd807aa98,0x12835b01,0x243185be,0x550c7dc3
        .long 0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174
        .long 0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc
        .long 0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da
        .long 0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7
        .long 0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967
        .long 0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13
        .long 0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85
        .long 0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3
        .long 0xd192e819,0xd6990624,0xf40e3585,0x106aa070
        .long 0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5
        .long 0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3
        .long 0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208
        .long 0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2