/* SPDX-License-Identifier: GPL-2.0-only */
/*
 * Accelerated CRC32(C) using AArch64 CRC and PMULL instructions
 *
 * Copyright (C) 2016 - 2018 Linaro Ltd.
 * Copyright (C) 2024 Google LLC
 *
 * Author: Ard Biesheuvel <ardb@kernel.org>
 */

#include <linux/linkage.h>
#include <asm/assembler.h>

        .cpu            generic+crc+crypto

        .macro          bitle, reg
        .endm

        .macro          bitbe, reg
        rbit            \reg, \reg
        .endm

        .macro          bytele, reg
        .endm

        .macro          bytebe, reg
        rbit            \reg, \reg
        lsr             \reg, \reg, #24
        .endm

        .macro          hwordle, reg
CPU_BE( rev16           \reg, \reg      )
        .endm

        .macro          hwordbe, reg
CPU_LE( rev             \reg, \reg      )
        rbit            \reg, \reg
CPU_BE( lsr             \reg, \reg, #16 )
        .endm

        .macro          le, regs:vararg
        .irp            r, \regs
CPU_BE( rev             \r, \r          )
        .endr
        .endm

        .macro          be, regs:vararg
        .irp            r, \regs
CPU_LE( rev             \r, \r          )
        .endr
        .irp            r, \regs
        rbit            \r, \r
        .endr
        .endm

        .macro          __crc32, c, order=le
        bit\order       w0
        cmp             x2, #16
        b.lt            8f                      // less than 16 bytes

        and             x7, x2, #0x1f
        and             x2, x2, #~0x1f
        cbz             x7, 32f                 // multiple of 32 bytes

        and             x8, x7, #0xf
        ldp             x3, x4, [x1]
        add             x8, x8, x1
        add             x1, x1, x7
        ldp             x5, x6, [x8]
        \order          x3, x4, x5, x6

        tst             x7, #8
        crc32\c\()x     w8, w0, x3
        csel            x3, x3, x4, eq
        csel            w0, w0, w8, eq
        tst             x7, #4
        lsr             x4, x3, #32
        crc32\c\()w     w8, w0, w3
        csel            x3, x3, x4, eq
        csel            w0, w0, w8, eq
        tst             x7, #2
        lsr             w4, w3, #16
        crc32\c\()h     w8, w0, w3
        csel            w3, w3, w4, eq
        csel            w0, w0, w8, eq
        tst             x7, #1
        crc32\c\()b     w8, w0, w3
        csel            w0, w0, w8, eq
        tst             x7, #16
        crc32\c\()x     w8, w0, x5
        crc32\c\()x     w8, w8, x6
        csel            w0, w0, w8, eq
        cbz             x2, 0f

32:     ldp             x3, x4, [x1], #32
        sub             x2, x2, #32
        ldp             x5, x6, [x1, #-16]
        \order          x3, x4, x5, x6
        crc32\c\()x     w0, w0, x3
        crc32\c\()x     w0, w0, x4
        crc32\c\()x     w0, w0, x5
        crc32\c\()x     w0, w0, x6
        cbnz            x2, 32b
0:      bit\order       w0
        ret

8:      tbz             x2, #3, 4f
        ldr             x3, [x1], #8
        \order          x3
        crc32\c\()x     w0, w0, x3
4:      tbz             x2, #2, 2f
        ldr             w3, [x1], #4
        \order          w3
        crc32\c\()w     w0, w0, w3
2:      tbz             x2, #1, 1f
        ldrh            w3, [x1], #2
        hword\order     w3
        crc32\c\()h     w0, w0, w3
1:      tbz             x2, #0, 0f
        ldrb            w3, [x1]
        byte\order      w3
        crc32\c\()b     w0, w0, w3
0:      bit\order       w0
        ret
        .endm

        .align          5
SYM_FUNC_START(crc32_le_arm64)
        __crc32
SYM_FUNC_END(crc32_le_arm64)

        .align          5
SYM_FUNC_START(crc32c_le_arm64)
        __crc32         c
SYM_FUNC_END(crc32c_le_arm64)

        .align          5
SYM_FUNC_START(crc32_be_arm64)
        __crc32         order=be
SYM_FUNC_END(crc32_be_arm64)

        in              .req    x1
        len             .req    x2

        /*
         * w0: input CRC at entry, output CRC at exit
         * x1: pointer to input buffer
         * x2: length of input in bytes
         */
        .macro          crc4way, insn, table, order=le
        bit\order       w0
        lsr             len, len, #6            // len := # of 64-byte blocks

        /* Process up to 64 blocks of 64 bytes at a time */
.La\@:  mov             x3, #64
        cmp             len, #64
        csel            x3, x3, len, hi         // x3 := min(len, 64)
        sub             len, len, x3

        /* Divide the input into 4 contiguous blocks */
        add             x4, x3, x3, lsl #1      // x4 :=  3 * x3
        add             x7, in, x3, lsl #4      // x7 := in + 16 * x3
        add             x8, in, x3, lsl #5      // x8 := in + 32 * x3
        add             x9, in, x4, lsl #4      // x9 := in + 16 * x4

        /* Load the folding coefficients from the lookup table */
        adr_l           x5, \table - 12         // entry 0 omitted
        add             x5, x5, x4, lsl #2      // x5 += 12 * x3
        ldp             s0, s1, [x5]
        ldr             s2, [x5, #8]

        /* Zero init partial CRCs for this iteration */
        mov             w4, wzr
        mov             w5, wzr
        mov             w6, wzr
        mov             x17, xzr

.Lb\@:  sub             x3, x3, #1
        \insn           w6, w6, x17
        ldp             x10, x11, [in], #16
        ldp             x12, x13, [x7], #16
        ldp             x14, x15, [x8], #16
        ldp             x16, x17, [x9], #16

        \order          x10, x11, x12, x13, x14, x15, x16, x17

        /* Apply the CRC transform to 4 16-byte blocks in parallel */
        \insn           w0, w0, x10
        \insn           w4, w4, x12
        \insn           w5, w5, x14
        \insn           w6, w6, x16
        \insn           w0, w0, x11
        \insn           w4, w4, x13
        \insn           w5, w5, x15
        cbnz            x3, .Lb\@

        /* Combine the 4 partial results into w0 */
        mov             v3.d[0], x0
        mov             v4.d[0], x4
        mov             v5.d[0], x5
        pmull           v0.1q, v0.1d, v3.1d
        pmull           v1.1q, v1.1d, v4.1d
        pmull           v2.1q, v2.1d, v5.1d
        eor             v0.8b, v0.8b, v1.8b
        eor             v0.8b, v0.8b, v2.8b
        mov             x5, v0.d[0]
        eor             x5, x5, x17
        \insn           w0, w6, x5

        mov             in, x9
        cbnz            len, .La\@

        bit\order       w0
        ret
        .endm

        .align          5
SYM_FUNC_START(crc32c_le_arm64_4way)
        crc4way         crc32cx, .L0
SYM_FUNC_END(crc32c_le_arm64_4way)

        .align          5
SYM_FUNC_START(crc32_le_arm64_4way)
        crc4way         crc32x, .L1
SYM_FUNC_END(crc32_le_arm64_4way)

        .align          5
SYM_FUNC_START(crc32_be_arm64_4way)
        crc4way         crc32x, .L1, be
SYM_FUNC_END(crc32_be_arm64_4way)

        .section        .rodata, "a", %progbits
        .align          6
.L0:    .long           0xddc0152b, 0xba4fc28e, 0x493c7d27
        .long           0x0715ce53, 0x9e4addf8, 0xba4fc28e
        .long           0xc96cfdc0, 0x0715ce53, 0xddc0152b
        .long           0xab7aff2a, 0x0d3b6092, 0x9e4addf8
        .long           0x299847d5, 0x878a92a7, 0x39d3b296
        .long           0xb6dd949b, 0xab7aff2a, 0x0715ce53
        .long           0xa60ce07b, 0x83348832, 0x47db8317
        .long           0xd270f1a2, 0xb9e02b86, 0x0d3b6092
        .long           0x65863b64, 0xb6dd949b, 0xc96cfdc0
        .long           0xb3e32c28, 0xbac2fd7b, 0x878a92a7
        .long           0xf285651c, 0xce7f39f4, 0xdaece73e
        .long           0x271d9844, 0xd270f1a2, 0xab7aff2a
        .long           0x6cb08e5c, 0x2b3cac5d, 0x2162d385
        .long           0xcec3662e, 0x1b03397f, 0x83348832
        .long           0x8227bb8a, 0xb3e32c28, 0x299847d5
        .long           0xd7a4825c, 0xdd7e3b0c, 0xb9e02b86
        .long           0xf6076544, 0x10746f3c, 0x18b33a4e
        .long           0x98d8d9cb, 0x271d9844, 0xb6dd949b
        .long           0x57a3d037, 0x93a5f730, 0x78d9ccb7
        .long           0x3771e98f, 0x6b749fb2, 0xbac2fd7b
        .long           0xe0ac139e, 0xcec3662e, 0xa60ce07b
        .long           0x6f345e45, 0xe6fc4e6a, 0xce7f39f4
        .long           0xa2b73df1, 0xb0cd4768, 0x61d82e56
        .long           0x86d8e4d2, 0xd7a4825c, 0xd270f1a2
        .long           0xa90fd27a, 0x0167d312, 0xc619809d
        .long           0xca6ef3ac, 0x26f6a60a, 0x2b3cac5d
        .long           0x4597456a, 0x98d8d9cb, 0x65863b64
        .long           0xc9c8b782, 0x68bce87a, 0x1b03397f
        .long           0x62ec6c6d, 0x6956fc3b, 0xebb883bd
        .long           0x2342001e, 0x3771e98f, 0xb3e32c28
        .long           0xe8b6368b, 0x2178513a, 0x064f7f26
        .long           0x9ef68d35, 0x170076fa, 0xdd7e3b0c
        .long           0x0b0bf8ca, 0x6f345e45, 0xf285651c
        .long           0x02ee03b2, 0xff0dba97, 0x10746f3c
        .long           0x135c83fd, 0xf872e54c, 0xc7a68855
        .long           0x00bcf5f6, 0x86d8e4d2, 0x271d9844
        .long           0x58ca5f00, 0x5bb8f1bc, 0x8e766a0c
        .long           0xded288f8, 0xb3af077a, 0x93a5f730
        .long           0x37170390, 0xca6ef3ac, 0x6cb08e5c
        .long           0xf48642e9, 0xdd66cbbb, 0x6b749fb2
        .long           0xb25b29f2, 0xe9e28eb4, 0x1393e203
        .long           0x45cddf4e, 0xc9c8b782, 0xcec3662e
        .long           0xdfd94fb2, 0x93e106a4, 0x96c515bb
        .long           0x021ac5ef, 0xd813b325, 0xe6fc4e6a
        .long           0x8e1450f7, 0x2342001e, 0x8227bb8a
        .long           0xe0cdcf86, 0x6d9a4957, 0xb0cd4768
        .long           0x613eee91, 0xd2c3ed1a, 0x39c7ff35
        .long           0xbedc6ba1, 0x9ef68d35, 0xd7a4825c
        .long           0x0cd1526a, 0xf2271e60, 0x0ab3844b
        .long           0xd6c3a807, 0x2664fd8b, 0x0167d312
        .long           0x1d31175f, 0x02ee03b2, 0xf6076544
        .long           0x4be7fd90, 0x363bd6b3, 0x26f6a60a
        .long           0x6eeed1c9, 0x5fabe670, 0xa741c1bf
        .long           0xb3a6da94, 0x00bcf5f6, 0x98d8d9cb
        .long           0x2e7d11a7, 0x17f27698, 0x49c3cc9c
        .long           0x889774e1, 0xaa7c7ad5, 0x68bce87a
        .long           0x8a074012, 0xded288f8, 0x57a3d037
        .long           0xbd0bb25f, 0x6d390dec, 0x6956fc3b
        .long           0x3be3c09b, 0x6353c1cc, 0x42d98888
        .long           0x465a4eee, 0xf48642e9, 0x3771e98f
        .long           0x2e5f3c8c, 0xdd35bc8d, 0xb42ae3d9
        .long           0xa52f58ec, 0x9a5ede41, 0x2178513a
        .long           0x47972100, 0x45cddf4e, 0xe0ac139e
        .long           0x359674f7, 0xa51b6135, 0x170076fa

.L1:    .long           0xaf449247, 0x81256527, 0xccaa009e
        .long           0x57c54819, 0x1d9513d7, 0x81256527
        .long           0x3f41287a, 0x57c54819, 0xaf449247
        .long           0xf5e48c85, 0x910eeec1, 0x1d9513d7
        .long           0x1f0c2cdd, 0x9026d5b1, 0xae0b5394
        .long           0x71d54a59, 0xf5e48c85, 0x57c54819
        .long           0x1c63267b, 0xfe807bbd, 0x0cbec0ed
        .long           0xd31343ea, 0xe95c1271, 0x910eeec1
        .long           0xf9d9c7ee, 0x71d54a59, 0x3f41287a
        .long           0x9ee62949, 0xcec97417, 0x9026d5b1
        .long           0xa55d1514, 0xf183c71b, 0xd1df2327
        .long           0x21aa2b26, 0xd31343ea, 0xf5e48c85
        .long           0x9d842b80, 0xeea395c4, 0x3c656ced
        .long           0xd8110ff1, 0xcd669a40, 0xfe807bbd
        .long           0x3f9e9356, 0x9ee62949, 0x1f0c2cdd
        .long           0x1d6708a0, 0x0c30f51d, 0xe95c1271
        .long           0xef82aa68, 0xdb3935ea, 0xb918a347
        .long           0xd14bcc9b, 0x21aa2b26, 0x71d54a59
        .long           0x99cce860, 0x356d209f, 0xff6f2fc2
        .long           0xd8af8e46, 0xc352f6de, 0xcec97417
        .long           0xf1996890, 0xd8110ff1, 0x1c63267b
        .long           0x631bc508, 0xe95c7216, 0xf183c71b
        .long           0x8511c306, 0x8e031a19, 0x9b9bdbd0
        .long           0xdb3839f3, 0x1d6708a0, 0xd31343ea
        .long           0x7a92fffb, 0xf7003835, 0x4470ac44
        .long           0x6ce68f2a, 0x00eba0c8, 0xeea395c4
        .long           0x4caaa263, 0xd14bcc9b, 0xf9d9c7ee
        .long           0xb46f7cff, 0x9a1b53c8, 0xcd669a40
        .long           0x60290934, 0x81b6f443, 0x6d40f445
        .long           0x8e976a7d, 0xd8af8e46, 0x9ee62949
        .long           0xdcf5088a, 0x9dbdc100, 0x145575d5
        .long           0x1753ab84, 0xbbf2f6d6, 0x0c30f51d
        .long           0x255b139e, 0x631bc508, 0xa55d1514
        .long           0xd784eaa8, 0xce26786c, 0xdb3935ea
        .long           0x6d2c864a, 0x8068c345, 0x2586d334
        .long           0x02072e24, 0xdb3839f3, 0x21aa2b26
        .long           0x06689b0a, 0x5efd72f5, 0xe0575528
        .long           0x1e52f5ea, 0x4117915b, 0x356d209f
        .long           0x1d3d1db6, 0x6ce68f2a, 0x9d842b80
        .long           0x3796455c, 0xb8e0e4a8, 0xc352f6de
        .long           0xdf3a4eb3, 0xc55a2330, 0xb84ffa9c
        .long           0x28ae0976, 0xb46f7cff, 0xd8110ff1
        .long           0x9764bc8d, 0xd7e7a22c, 0x712510f0
        .long           0x13a13e18, 0x3e9a43cd, 0xe95c7216
        .long           0xb8ee242e, 0x8e976a7d, 0x3f9e9356
        .long           0x0c540e7b, 0x753c81ff, 0x8e031a19
        .long           0x9924c781, 0xb9220208, 0x3edcde65
        .long           0x3954de39, 0x1753ab84, 0x1d6708a0
        .long           0xf32238b5, 0xbec81497, 0x9e70b943
        .long           0xbbd2cd2c, 0x0925d861, 0xf7003835
        .long           0xcc401304, 0xd784eaa8, 0xef82aa68
        .long           0x4987e684, 0x6044fbb0, 0x00eba0c8
        .long           0x3aa11427, 0x18fe3b4a, 0x87441142
        .long           0x297aad60, 0x02072e24, 0xd14bcc9b
        .long           0xf60c5e51, 0x6ef6f487, 0x5b7fdd0a
        .long           0x632d78c5, 0x3fc33de4, 0x9a1b53c8
        .long           0x25b8822a, 0x1e52f5ea, 0x99cce860
        .long           0xd4fc84bc, 0x1af62fb8, 0x81b6f443
        .long           0x5690aa32, 0xa91fdefb, 0x688a110e
        .long           0x1357a093, 0x3796455c, 0xd8af8e46
        .long           0x798fdd33, 0xaaa18a37, 0x357b9517
        .long           0xc2815395, 0x54d42691, 0x9dbdc100
        .long           0x21cfc0f7, 0x28ae0976, 0xf1996890
        .long           0xa0decef3, 0x7b4aa8b7, 0xbbf2f6d6