/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * sm3-ce-core.S - SM3 secure hash using ARMv8.2 Crypto Extensions
 *
 * Copyright (C) 2018 Linaro Ltd <ard.biesheuvel@linaro.org>
 */

#include <linux/linkage.h>
#include <asm/assembler.h>

        .irp            b, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12
        .set            .Lv\b\().4s, \b
        .endr

        .macro          sm3partw1, rd, rn, rm
        .inst           0xce60c000 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
        .endm

        .macro          sm3partw2, rd, rn, rm
        .inst           0xce60c400 | .L\rd | (.L\rn << 5) | (.L\rm << 16)
        .endm

        .macro          sm3ss1, rd, rn, rm, ra
        .inst           0xce400000 | .L\rd | (.L\rn << 5) | (.L\ra << 10) | (.L\rm << 16)
        .endm

        .macro          sm3tt1a, rd, rn, rm, imm2
        .inst           0xce408000 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
        .endm

        .macro          sm3tt1b, rd, rn, rm, imm2
        .inst           0xce408400 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
        .endm

        .macro          sm3tt2a, rd, rn, rm, imm2
        .inst           0xce408800 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
        .endm

        .macro          sm3tt2b, rd, rn, rm, imm2
        .inst           0xce408c00 | .L\rd | (.L\rn << 5) | ((\imm2) << 12) | (.L\rm << 16)
        .endm

        .macro          round, ab, s0, t0, t1, i
        sm3ss1          v5.4s, v8.4s, \t0\().4s, v9.4s
        shl             \t1\().4s, \t0\().4s, #1
        sri             \t1\().4s, \t0\().4s, #31
        sm3tt1\ab       v8.4s, v5.4s, v10.4s, \i
        sm3tt2\ab       v9.4s, v5.4s, \s0\().4s, \i
        .endm

        .macro          qround, ab, s0, s1, s2, s3, s4
        .ifnb           \s4
        ext             \s4\().16b, \s1\().16b, \s2\().16b, #12
        ext             v6.16b, \s0\().16b, \s1\().16b, #12
        ext             v7.16b, \s2\().16b, \s3\().16b, #8
        sm3partw1       \s4\().4s, \s0\().4s, \s3\().4s
        .endif

        eor             v10.16b, \s0\().16b, \s1\().16b

        round           \ab, \s0, v11, v12, 0
        round           \ab, \s0, v12, v11, 1
        round           \ab, \s0, v11, v12, 2
        round           \ab, \s0, v12, v11, 3

        .ifnb           \s4
        sm3partw2       \s4\().4s, v7.4s, v6.4s
        .endif
        .endm

        /*
         * void sm3_ce_transform(struct sm3_block_state *state,
         *                       const u8 *data, size_t nblocks)
         */
        .text
SYM_FUNC_START(sm3_ce_transform)
        /* load state */
        ld1             {v8.4s-v9.4s}, [x0]
        rev64           v8.4s, v8.4s
        rev64           v9.4s, v9.4s
        ext             v8.16b, v8.16b, v8.16b, #8
        ext             v9.16b, v9.16b, v9.16b, #8

        adr_l           x8, .Lt
        ldp             s13, s14, [x8]

        /* load input */
0:      ld1             {v0.16b-v3.16b}, [x1], #64
        sub             x2, x2, #1

        mov             v15.16b, v8.16b
        mov             v16.16b, v9.16b

        rev32           v0.16b, v0.16b
        rev32           v1.16b, v1.16b
        rev32           v2.16b, v2.16b
        rev32           v3.16b, v3.16b

        ext             v11.16b, v13.16b, v13.16b, #4

        qround          a, v0, v1, v2, v3, v4
        qround          a, v1, v2, v3, v4, v0
        qround          a, v2, v3, v4, v0, v1
        qround          a, v3, v4, v0, v1, v2

        ext             v11.16b, v14.16b, v14.16b, #4

        qround          b, v4, v0, v1, v2, v3
        qround          b, v0, v1, v2, v3, v4
        qround          b, v1, v2, v3, v4, v0
        qround          b, v2, v3, v4, v0, v1
        qround          b, v3, v4, v0, v1, v2
        qround          b, v4, v0, v1, v2, v3
        qround          b, v0, v1, v2, v3, v4
        qround          b, v1, v2, v3, v4, v0
        qround          b, v2, v3, v4, v0, v1
        qround          b, v3, v4
        qround          b, v4, v0
        qround          b, v0, v1

        eor             v8.16b, v8.16b, v15.16b
        eor             v9.16b, v9.16b, v16.16b

        /* handled all input blocks? */
        cbnz            x2, 0b

        /* save state */
        rev64           v8.4s, v8.4s
        rev64           v9.4s, v9.4s
        ext             v8.16b, v8.16b, v8.16b, #8
        ext             v9.16b, v9.16b, v9.16b, #8
        st1             {v8.4s-v9.4s}, [x0]
        ret
SYM_FUNC_END(sm3_ce_transform)

        .section        ".rodata", "a"
        .align          3
.Lt:    .word           0x79cc4519, 0x9d8a7a87