/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 * SM4-CCM AEAD Algorithm using ARMv8 Crypto Extensions
 * as specified in rfc8998
 * https://datatracker.ietf.org/doc/html/rfc8998
 *
 * Copyright (C) 2022 Tianjia Zhang <tianjia.zhang@linux.alibaba.com>
 */

#include <linux/linkage.h>
#include <linux/cfi_types.h>
#include <asm/assembler.h>
#include "sm4-ce-asm.h"

.arch   armv8-a+crypto

.irp b, 0, 1, 8, 9, 10, 11, 12, 13, 14, 15, 16, 24, 25, 26, 27, 28, 29, 30, 31
        .set .Lv\b\().4s, \b
.endr

.macro sm4e, vd, vn
        .inst 0xcec08400 | (.L\vn << 5) | .L\vd
.endm

/* Register macros */

#define RMAC    v16

/* Helper macros. */

#define inc_le128(vctr)                                 \
                mov             vctr.d[1], x8;          \
                mov             vctr.d[0], x7;          \
                adds            x8, x8, #1;             \
                rev64           vctr.16b, vctr.16b;     \
                adc             x7, x7, xzr;


.align 3
SYM_FUNC_START(sm4_ce_cbcmac_update)
        /* input:
         *   x0: round key array, CTX
         *   x1: mac
         *   x2: src
         *   w3: nblocks
         */
        SM4_PREPARE(x0)

        ld1             {RMAC.16b}, [x1]

.Lcbcmac_loop_4x:
        cmp             w3, #4
        blt             .Lcbcmac_loop_1x

        sub             w3, w3, #4

        ld1             {v0.16b-v3.16b}, [x2], #64

        SM4_CRYPT_BLK(RMAC)
        eor             RMAC.16b, RMAC.16b, v0.16b
        SM4_CRYPT_BLK(RMAC)
        eor             RMAC.16b, RMAC.16b, v1.16b
        SM4_CRYPT_BLK(RMAC)
        eor             RMAC.16b, RMAC.16b, v2.16b
        SM4_CRYPT_BLK(RMAC)
        eor             RMAC.16b, RMAC.16b, v3.16b

        cbz             w3, .Lcbcmac_end
        b               .Lcbcmac_loop_4x

.Lcbcmac_loop_1x:
        sub             w3, w3, #1

        ld1             {v0.16b}, [x2], #16

        SM4_CRYPT_BLK(RMAC)
        eor             RMAC.16b, RMAC.16b, v0.16b

        cbnz            w3, .Lcbcmac_loop_1x

.Lcbcmac_end:
        st1             {RMAC.16b}, [x1]
        ret
SYM_FUNC_END(sm4_ce_cbcmac_update)

.align 3
SYM_FUNC_START(sm4_ce_ccm_final)
        /* input:
         *   x0: round key array, CTX
         *   x1: ctr0 (big endian, 128 bit)
         *   x2: mac
         */
        SM4_PREPARE(x0)

        ld1             {RMAC.16b}, [x2]
        ld1             {v0.16b}, [x1]

        SM4_CRYPT_BLK2(RMAC, v0)

        /* en-/decrypt the mac with ctr0 */
        eor             RMAC.16b, RMAC.16b, v0.16b
        st1             {RMAC.16b}, [x2]

        ret
SYM_FUNC_END(sm4_ce_ccm_final)

.align 3
SYM_TYPED_FUNC_START(sm4_ce_ccm_enc)
        /* input:
         *   x0: round key array, CTX
         *   x1: dst
         *   x2: src
         *   x3: ctr (big endian, 128 bit)
         *   w4: nbytes
         *   x5: mac
         */
        SM4_PREPARE(x0)

        ldp             x7, x8, [x3]
        rev             x7, x7
        rev             x8, x8

        ld1             {RMAC.16b}, [x5]

.Lccm_enc_loop_4x:
        cmp             w4, #(4 * 16)
        blt             .Lccm_enc_loop_1x

        sub             w4, w4, #(4 * 16)

        /* construct CTRs */
        inc_le128(v8)                   /* +0 */
        inc_le128(v9)                   /* +1 */
        inc_le128(v10)                  /* +2 */
        inc_le128(v11)                  /* +3 */

        ld1             {v0.16b-v3.16b}, [x2], #64

        SM4_CRYPT_BLK2(v8, RMAC)
        eor             v8.16b, v8.16b, v0.16b
        eor             RMAC.16b, RMAC.16b, v0.16b
        SM4_CRYPT_BLK2(v9, RMAC)
        eor             v9.16b, v9.16b, v1.16b
        eor             RMAC.16b, RMAC.16b, v1.16b
        SM4_CRYPT_BLK2(v10, RMAC)
        eor             v10.16b, v10.16b, v2.16b
        eor             RMAC.16b, RMAC.16b, v2.16b
        SM4_CRYPT_BLK2(v11, RMAC)
        eor             v11.16b, v11.16b, v3.16b
        eor             RMAC.16b, RMAC.16b, v3.16b

        st1             {v8.16b-v11.16b}, [x1], #64

        cbz             w4, .Lccm_enc_end
        b               .Lccm_enc_loop_4x

.Lccm_enc_loop_1x:
        cmp             w4, #16
        blt             .Lccm_enc_tail

        sub             w4, w4, #16

        /* construct CTRs */
        inc_le128(v8)

        ld1             {v0.16b}, [x2], #16

        SM4_CRYPT_BLK2(v8, RMAC)
        eor             v8.16b, v8.16b, v0.16b
        eor             RMAC.16b, RMAC.16b, v0.16b

        st1             {v8.16b}, [x1], #16

        cbz             w4, .Lccm_enc_end
        b               .Lccm_enc_loop_1x

.Lccm_enc_tail:
        /* construct CTRs */
        inc_le128(v8)

        SM4_CRYPT_BLK2(RMAC, v8)

        /* store new MAC */
        st1             {RMAC.16b}, [x5]

.Lccm_enc_tail_loop:
        ldrb            w0, [x2], #1            /* get 1 byte from input */
        umov            w9, v8.b[0]             /* get top crypted CTR byte */
        umov            w6, RMAC.b[0]           /* get top MAC byte */

        eor             w9, w9, w0              /* w9 = CTR ^ input */
        eor             w6, w6, w0              /* w6 = MAC ^ input */

        strb            w9, [x1], #1            /* store out byte */
        strb            w6, [x5], #1            /* store MAC byte */

        subs            w4, w4, #1
        beq             .Lccm_enc_ret

        /* shift out one byte */
        ext             RMAC.16b, RMAC.16b, RMAC.16b, #1
        ext             v8.16b, v8.16b, v8.16b, #1

        b               .Lccm_enc_tail_loop

.Lccm_enc_end:
        /* store new MAC */
        st1             {RMAC.16b}, [x5]

        /* store new CTR */
        rev             x7, x7
        rev             x8, x8
        stp             x7, x8, [x3]

.Lccm_enc_ret:
        ret
SYM_FUNC_END(sm4_ce_ccm_enc)

.align 3
SYM_TYPED_FUNC_START(sm4_ce_ccm_dec)
        /* input:
         *   x0: round key array, CTX
         *   x1: dst
         *   x2: src
         *   x3: ctr (big endian, 128 bit)
         *   w4: nbytes
         *   x5: mac
         */
        SM4_PREPARE(x0)

        ldp             x7, x8, [x3]
        rev             x7, x7
        rev             x8, x8

        ld1             {RMAC.16b}, [x5]

.Lccm_dec_loop_4x:
        cmp             w4, #(4 * 16)
        blt             .Lccm_dec_loop_1x

        sub             w4, w4, #(4 * 16)

        /* construct CTRs */
        inc_le128(v8)                   /* +0 */
        inc_le128(v9)                   /* +1 */
        inc_le128(v10)                  /* +2 */
        inc_le128(v11)                  /* +3 */

        ld1             {v0.16b-v3.16b}, [x2], #64

        SM4_CRYPT_BLK2(v8, RMAC)
        eor             v8.16b, v8.16b, v0.16b
        eor             RMAC.16b, RMAC.16b, v8.16b
        SM4_CRYPT_BLK2(v9, RMAC)
        eor             v9.16b, v9.16b, v1.16b
        eor             RMAC.16b, RMAC.16b, v9.16b
        SM4_CRYPT_BLK2(v10, RMAC)
        eor             v10.16b, v10.16b, v2.16b
        eor             RMAC.16b, RMAC.16b, v10.16b
        SM4_CRYPT_BLK2(v11, RMAC)
        eor             v11.16b, v11.16b, v3.16b
        eor             RMAC.16b, RMAC.16b, v11.16b

        st1             {v8.16b-v11.16b}, [x1], #64

        cbz             w4, .Lccm_dec_end
        b               .Lccm_dec_loop_4x

.Lccm_dec_loop_1x:
        cmp             w4, #16
        blt             .Lccm_dec_tail

        sub             w4, w4, #16

        /* construct CTRs */
        inc_le128(v8)

        ld1             {v0.16b}, [x2], #16

        SM4_CRYPT_BLK2(v8, RMAC)
        eor             v8.16b, v8.16b, v0.16b
        eor             RMAC.16b, RMAC.16b, v8.16b

        st1             {v8.16b}, [x1], #16

        cbz             w4, .Lccm_dec_end
        b               .Lccm_dec_loop_1x

.Lccm_dec_tail:
        /* construct CTRs */
        inc_le128(v8)

        SM4_CRYPT_BLK2(RMAC, v8)

        /* store new MAC */
        st1             {RMAC.16b}, [x5]

.Lccm_dec_tail_loop:
        ldrb            w0, [x2], #1            /* get 1 byte from input */
        umov            w9, v8.b[0]             /* get top crypted CTR byte */
        umov            w6, RMAC.b[0]           /* get top MAC byte */

        eor             w9, w9, w0              /* w9 = CTR ^ input */
        eor             w6, w6, w9              /* w6 = MAC ^ output */

        strb            w9, [x1], #1            /* store out byte */
        strb            w6, [x5], #1            /* store MAC byte */

        subs            w4, w4, #1
        beq             .Lccm_dec_ret

        /* shift out one byte */
        ext             RMAC.16b, RMAC.16b, RMAC.16b, #1
        ext             v8.16b, v8.16b, v8.16b, #1

        b               .Lccm_dec_tail_loop

.Lccm_dec_end:
        /* store new MAC */
        st1             {RMAC.16b}, [x5]

        /* store new CTR */
        rev             x7, x7
        rev             x8, x8
        stp             x7, x8, [x3]

.Lccm_dec_ret:
        ret
SYM_FUNC_END(sm4_ce_ccm_dec)