/* SPDX-License-Identifier: GPL-2.0 */

#include <linux/stringify.h>
#include <linux/linkage.h>
#include <asm/alternative.h>
#include <asm/dwarf.h>
#include <asm/fpu-insn.h>

#define STATE0  %v0
#define STATE1  %v1
#define STATE2  %v2
#define STATE3  %v3
#define COPY0   %v4
#define COPY1   %v5
#define COPY2   %v6
#define COPY3   %v7
#define BEPERM  %v19
#define TMP0    %v20
#define TMP1    %v21
#define TMP2    %v22
#define TMP3    %v23

        .section .rodata

        .balign 32
SYM_DATA_START_LOCAL(chacha20_constants)
        .long   0x61707865,0x3320646e,0x79622d32,0x6b206574 # endian-neutral
        .long   0x03020100,0x07060504,0x0b0a0908,0x0f0e0d0c # byte swap
SYM_DATA_END(chacha20_constants)

        .text
/*
 * s390 ChaCha20 implementation meant for vDSO. Produces a given positive
 * number of blocks of output with nonce 0, taking an input key and 8-bytes
 * counter. Does not spill to the stack.
 *
 * void __arch_chacha20_blocks_nostack(uint8_t *dst_bytes,
 *                                     const uint8_t *key,
 *                                     uint32_t *counter,
 *                                     size_t nblocks)
 */
SYM_FUNC_START(__arch_chacha20_blocks_nostack)
        CFI_STARTPROC
        larl    %r1,chacha20_constants

        /* COPY0 = "expand 32-byte k" */
        VL      COPY0,0,,%r1

        /* BEPERM = byte selectors for VPERM */
        ALTERNATIVE __stringify(VL BEPERM,16,,%r1), "brcl 0,0", ALT_FACILITY(148)

        /* COPY1,COPY2 = key */
        VLM     COPY1,COPY2,0,%r3

        /* COPY3 = counter || zero nonce  */
        lg      %r3,0(%r4)
        VZERO   COPY3
        VLVGG   COPY3,%r3,0

        lghi    %r1,0
.Lblock:
        VLR     STATE0,COPY0
        VLR     STATE1,COPY1
        VLR     STATE2,COPY2
        VLR     STATE3,COPY3

        lghi    %r0,10
.Ldoubleround:
        /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 16) */
        VAF     STATE0,STATE0,STATE1
        VX      STATE3,STATE3,STATE0
        VERLLF  STATE3,STATE3,16

        /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 12) */
        VAF     STATE2,STATE2,STATE3
        VX      STATE1,STATE1,STATE2
        VERLLF  STATE1,STATE1,12

        /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 8) */
        VAF     STATE0,STATE0,STATE1
        VX      STATE3,STATE3,STATE0
        VERLLF  STATE3,STATE3,8

        /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 7) */
        VAF     STATE2,STATE2,STATE3
        VX      STATE1,STATE1,STATE2
        VERLLF  STATE1,STATE1,7

        /* STATE1[0,1,2,3] = STATE1[1,2,3,0] */
        VSLDB   STATE1,STATE1,STATE1,4
        /* STATE2[0,1,2,3] = STATE2[2,3,0,1] */
        VSLDB   STATE2,STATE2,STATE2,8
        /* STATE3[0,1,2,3] = STATE3[3,0,1,2] */
        VSLDB   STATE3,STATE3,STATE3,12

        /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 16) */
        VAF     STATE0,STATE0,STATE1
        VX      STATE3,STATE3,STATE0
        VERLLF  STATE3,STATE3,16

        /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 12) */
        VAF     STATE2,STATE2,STATE3
        VX      STATE1,STATE1,STATE2
        VERLLF  STATE1,STATE1,12

        /* STATE0 += STATE1, STATE3 = rotl32(STATE3 ^ STATE0, 8) */
        VAF     STATE0,STATE0,STATE1
        VX      STATE3,STATE3,STATE0
        VERLLF  STATE3,STATE3,8

        /* STATE2 += STATE3, STATE1 = rotl32(STATE1 ^ STATE2, 7) */
        VAF     STATE2,STATE2,STATE3
        VX      STATE1,STATE1,STATE2
        VERLLF  STATE1,STATE1,7

        /* STATE1[0,1,2,3] = STATE1[3,0,1,2] */
        VSLDB   STATE1,STATE1,STATE1,12
        /* STATE2[0,1,2,3] = STATE2[2,3,0,1] */
        VSLDB   STATE2,STATE2,STATE2,8
        /* STATE3[0,1,2,3] = STATE3[1,2,3,0] */
        VSLDB   STATE3,STATE3,STATE3,4
        brctg   %r0,.Ldoubleround

        /* OUTPUT0 = STATE0 + COPY0 */
        VAF     STATE0,STATE0,COPY0
        /* OUTPUT1 = STATE1 + COPY1 */
        VAF     STATE1,STATE1,COPY1
        /* OUTPUT2 = STATE2 + COPY2 */
        VAF     STATE2,STATE2,COPY2
        /* OUTPUT3 = STATE3 + COPY3 */
        VAF     STATE3,STATE3,COPY3

        ALTERNATIVE                                                     \
                __stringify(                                            \
                /* Convert STATE to little endian and store to OUTPUT */\
                VPERM   TMP0,STATE0,STATE0,BEPERM;                      \
                VPERM   TMP1,STATE1,STATE1,BEPERM;                      \
                VPERM   TMP2,STATE2,STATE2,BEPERM;                      \
                VPERM   TMP3,STATE3,STATE3,BEPERM;                      \
                VSTM    TMP0,TMP3,0,%r2),                               \
                __stringify(                                            \
                /* 32 bit wise little endian store to OUTPUT */         \
                VSTBRF  STATE0,0,,%r2;                                  \
                VSTBRF  STATE1,16,,%r2;                                 \
                VSTBRF  STATE2,32,,%r2;                                 \
                VSTBRF  STATE3,48,,%r2;                                 \
                brcl    0,0),                                           \
                ALT_FACILITY(148)

        /* ++COPY3.COUNTER */
        /* alsih %r3,1 */
        .insn   rilu,0xcc0a00000000,%r3,1
        alcr    %r3,%r1
        VLVGG   COPY3,%r3,0

        /* OUTPUT += 64, --NBLOCKS */
        aghi    %r2,64
        brctg   %r5,.Lblock

        /* COUNTER = COPY3.COUNTER */
        stg     %r3,0(%r4)

        /* Zero out potentially sensitive regs */
        VZERO   STATE0
        VZERO   STATE1
        VZERO   STATE2
        VZERO   STATE3
        VZERO   COPY1
        VZERO   COPY2

        /* Early exit if TMP0-TMP3 have not been used */
        ALTERNATIVE "nopr", "br %r14", ALT_FACILITY(148)

        VZERO   TMP0
        VZERO   TMP1
        VZERO   TMP2
        VZERO   TMP3

        br      %r14
        CFI_ENDPROC
SYM_FUNC_END(__arch_chacha20_blocks_nostack)