/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * linux/arch/arm64/crypto/aes-neon.S - AES cipher for ARMv8 NEON
 *
 * Copyright (C) 2013 - 2017 Linaro Ltd. <ard.biesheuvel@linaro.org>
 */

#include <linux/linkage.h>
#include <asm/assembler.h>

#define AES_FUNC_START(func)            SYM_FUNC_START(neon_ ## func)
#define AES_FUNC_END(func)              SYM_FUNC_END(neon_ ## func)

        xtsmask         .req    v7
        cbciv           .req    v7
        vctr            .req    v4

        .macro          xts_reload_mask, tmp
        xts_load_mask   \tmp
        .endm

        /* special case for the neon-bs driver calling into this one for CTS */
        .macro          xts_cts_skip_tw, reg, lbl
        tbnz            \reg, #1, \lbl
        .endm

        /* multiply by polynomial 'x' in GF(2^8) */
        .macro          mul_by_x, out, in, temp, const
        sshr            \temp, \in, #7
        shl             \out, \in, #1
        and             \temp, \temp, \const
        eor             \out, \out, \temp
        .endm

        /* multiply by polynomial 'x^2' in GF(2^8) */
        .macro          mul_by_x2, out, in, temp, const
        ushr            \temp, \in, #6
        shl             \out, \in, #2
        pmul            \temp, \temp, \const
        eor             \out, \out, \temp
        .endm

        /* preload the entire Sbox */
        .macro          prepare, sbox, shiftrows, temp
        movi            v12.16b, #0x1b
        ldr_l           q13, \shiftrows, \temp
        ldr_l           q14, .Lror32by8, \temp
        adr_l           \temp, \sbox
        ld1             {v16.16b-v19.16b}, [\temp], #64
        ld1             {v20.16b-v23.16b}, [\temp], #64
        ld1             {v24.16b-v27.16b}, [\temp], #64
        ld1             {v28.16b-v31.16b}, [\temp]
        .endm

        /* do preload for encryption */
        .macro          enc_prepare, ignore0, ignore1, temp
        prepare         crypto_aes_sbox, .LForward_ShiftRows, \temp
        .endm

        .macro          enc_switch_key, ignore0, ignore1, temp
        /* do nothing */
        .endm

        /* do preload for decryption */
        .macro          dec_prepare, ignore0, ignore1, temp
        prepare         crypto_aes_inv_sbox, .LReverse_ShiftRows, \temp
        .endm

        /* apply SubBytes transformation using the preloaded Sbox */
        .macro          sub_bytes, in
        sub             v9.16b, \in\().16b, v15.16b
        tbl             \in\().16b, {v16.16b-v19.16b}, \in\().16b
        sub             v10.16b, v9.16b, v15.16b
        tbx             \in\().16b, {v20.16b-v23.16b}, v9.16b
        sub             v11.16b, v10.16b, v15.16b
        tbx             \in\().16b, {v24.16b-v27.16b}, v10.16b
        tbx             \in\().16b, {v28.16b-v31.16b}, v11.16b
        .endm

        /* apply MixColumns transformation */
        .macro          mix_columns, in, enc
        .if             \enc == 0
        /* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
        mul_by_x2       v8.16b, \in\().16b, v9.16b, v12.16b
        eor             \in\().16b, \in\().16b, v8.16b
        rev32           v8.8h, v8.8h
        eor             \in\().16b, \in\().16b, v8.16b
        .endif

        mul_by_x        v9.16b, \in\().16b, v8.16b, v12.16b
        rev32           v8.8h, \in\().8h
        eor             v8.16b, v8.16b, v9.16b
        eor             \in\().16b, \in\().16b, v8.16b
        tbl             \in\().16b, {\in\().16b}, v14.16b
        eor             \in\().16b, \in\().16b, v8.16b
        .endm

        .macro          do_block, enc, in, rounds, rk, rkp, i
        ld1             {v15.4s}, [\rk]
        add             \rkp, \rk, #16
        mov             \i, \rounds
.La\@:  eor             \in\().16b, \in\().16b, v15.16b         /* ^round key */
        movi            v15.16b, #0x40
        tbl             \in\().16b, {\in\().16b}, v13.16b       /* ShiftRows */
        sub_bytes       \in
        sub             \i, \i, #1
        ld1             {v15.4s}, [\rkp], #16
        cbz             \i, .Lb\@
        mix_columns     \in, \enc
        b               .La\@
.Lb\@:  eor             \in\().16b, \in\().16b, v15.16b         /* ^round key */
        .endm

        .macro          encrypt_block, in, rounds, rk, rkp, i
        do_block        1, \in, \rounds, \rk, \rkp, \i
        .endm

        .macro          decrypt_block, in, rounds, rk, rkp, i
        do_block        0, \in, \rounds, \rk, \rkp, \i
        .endm

        /*
         * Interleaved versions: functionally equivalent to the
         * ones above, but applied to AES states in parallel.
         */

        .macro          sub_bytes_4x, in0, in1, in2, in3
        sub             v8.16b, \in0\().16b, v15.16b
        tbl             \in0\().16b, {v16.16b-v19.16b}, \in0\().16b
        sub             v9.16b, \in1\().16b, v15.16b
        tbl             \in1\().16b, {v16.16b-v19.16b}, \in1\().16b
        sub             v10.16b, \in2\().16b, v15.16b
        tbl             \in2\().16b, {v16.16b-v19.16b}, \in2\().16b
        sub             v11.16b, \in3\().16b, v15.16b
        tbl             \in3\().16b, {v16.16b-v19.16b}, \in3\().16b
        tbx             \in0\().16b, {v20.16b-v23.16b}, v8.16b
        tbx             \in1\().16b, {v20.16b-v23.16b}, v9.16b
        sub             v8.16b, v8.16b, v15.16b
        tbx             \in2\().16b, {v20.16b-v23.16b}, v10.16b
        sub             v9.16b, v9.16b, v15.16b
        tbx             \in3\().16b, {v20.16b-v23.16b}, v11.16b
        sub             v10.16b, v10.16b, v15.16b
        tbx             \in0\().16b, {v24.16b-v27.16b}, v8.16b
        sub             v11.16b, v11.16b, v15.16b
        tbx             \in1\().16b, {v24.16b-v27.16b}, v9.16b
        sub             v8.16b, v8.16b, v15.16b
        tbx             \in2\().16b, {v24.16b-v27.16b}, v10.16b
        sub             v9.16b, v9.16b, v15.16b
        tbx             \in3\().16b, {v24.16b-v27.16b}, v11.16b
        sub             v10.16b, v10.16b, v15.16b
        tbx             \in0\().16b, {v28.16b-v31.16b}, v8.16b
        sub             v11.16b, v11.16b, v15.16b
        tbx             \in1\().16b, {v28.16b-v31.16b}, v9.16b
        tbx             \in2\().16b, {v28.16b-v31.16b}, v10.16b
        tbx             \in3\().16b, {v28.16b-v31.16b}, v11.16b
        .endm

        .macro          mul_by_x_2x, out0, out1, in0, in1, tmp0, tmp1, const
        sshr            \tmp0\().16b, \in0\().16b, #7
        shl             \out0\().16b, \in0\().16b, #1
        sshr            \tmp1\().16b, \in1\().16b, #7
        and             \tmp0\().16b, \tmp0\().16b, \const\().16b
        shl             \out1\().16b, \in1\().16b, #1
        and             \tmp1\().16b, \tmp1\().16b, \const\().16b
        eor             \out0\().16b, \out0\().16b, \tmp0\().16b
        eor             \out1\().16b, \out1\().16b, \tmp1\().16b
        .endm

        .macro          mul_by_x2_2x, out0, out1, in0, in1, tmp0, tmp1, const
        ushr            \tmp0\().16b, \in0\().16b, #6
        shl             \out0\().16b, \in0\().16b, #2
        ushr            \tmp1\().16b, \in1\().16b, #6
        pmul            \tmp0\().16b, \tmp0\().16b, \const\().16b
        shl             \out1\().16b, \in1\().16b, #2
        pmul            \tmp1\().16b, \tmp1\().16b, \const\().16b
        eor             \out0\().16b, \out0\().16b, \tmp0\().16b
        eor             \out1\().16b, \out1\().16b, \tmp1\().16b
        .endm

        .macro          mix_columns_2x, in0, in1, enc
        .if             \enc == 0
        /* Inverse MixColumns: pre-multiply by { 5, 0, 4, 0 } */
        mul_by_x2_2x    v8, v9, \in0, \in1, v10, v11, v12
        eor             \in0\().16b, \in0\().16b, v8.16b
        rev32           v8.8h, v8.8h
        eor             \in1\().16b, \in1\().16b, v9.16b
        rev32           v9.8h, v9.8h
        eor             \in0\().16b, \in0\().16b, v8.16b
        eor             \in1\().16b, \in1\().16b, v9.16b
        .endif

        mul_by_x_2x     v8, v9, \in0, \in1, v10, v11, v12
        rev32           v10.8h, \in0\().8h
        rev32           v11.8h, \in1\().8h
        eor             v10.16b, v10.16b, v8.16b
        eor             v11.16b, v11.16b, v9.16b
        eor             \in0\().16b, \in0\().16b, v10.16b
        eor             \in1\().16b, \in1\().16b, v11.16b
        tbl             \in0\().16b, {\in0\().16b}, v14.16b
        tbl             \in1\().16b, {\in1\().16b}, v14.16b
        eor             \in0\().16b, \in0\().16b, v10.16b
        eor             \in1\().16b, \in1\().16b, v11.16b
        .endm

        .macro          do_block_4x, enc, in0, in1, in2, in3, rounds, rk, rkp, i
        ld1             {v15.4s}, [\rk]
        add             \rkp, \rk, #16
        mov             \i, \rounds
.La\@:  eor             \in0\().16b, \in0\().16b, v15.16b       /* ^round key */
        eor             \in1\().16b, \in1\().16b, v15.16b       /* ^round key */
        eor             \in2\().16b, \in2\().16b, v15.16b       /* ^round key */
        eor             \in3\().16b, \in3\().16b, v15.16b       /* ^round key */
        movi            v15.16b, #0x40
        tbl             \in0\().16b, {\in0\().16b}, v13.16b     /* ShiftRows */
        tbl             \in1\().16b, {\in1\().16b}, v13.16b     /* ShiftRows */
        tbl             \in2\().16b, {\in2\().16b}, v13.16b     /* ShiftRows */
        tbl             \in3\().16b, {\in3\().16b}, v13.16b     /* ShiftRows */
        sub_bytes_4x    \in0, \in1, \in2, \in3
        sub             \i, \i, #1
        ld1             {v15.4s}, [\rkp], #16
        cbz             \i, .Lb\@
        mix_columns_2x  \in0, \in1, \enc
        mix_columns_2x  \in2, \in3, \enc
        b               .La\@
.Lb\@:  eor             \in0\().16b, \in0\().16b, v15.16b       /* ^round key */
        eor             \in1\().16b, \in1\().16b, v15.16b       /* ^round key */
        eor             \in2\().16b, \in2\().16b, v15.16b       /* ^round key */
        eor             \in3\().16b, \in3\().16b, v15.16b       /* ^round key */
        .endm

        .macro          encrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
        do_block_4x     1, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
        .endm

        .macro          decrypt_block4x, in0, in1, in2, in3, rounds, rk, rkp, i
        do_block_4x     0, \in0, \in1, \in2, \in3, \rounds, \rk, \rkp, \i
        .endm

#include "aes-modes.S"

        .section        ".rodata", "a"
        .align          4
.LForward_ShiftRows:
        .octa           0x0b06010c07020d08030e09040f0a0500

.LReverse_ShiftRows:
        .octa           0x0306090c0f0205080b0e0104070a0d00

.Lror32by8:
        .octa           0x0c0f0e0d080b0a090407060500030201