/* SPDX-License-Identifier: GPL-2.0 */
/*
 * NH - ε-almost-universal hash function, NEON accelerated version
 *
 * Copyright 2018 Google LLC
 *
 * Author: Eric Biggers <ebiggers@google.com>
 */

#include <linux/linkage.h>

        .text
        .fpu            neon

        KEY             .req    r0
        MESSAGE         .req    r1
        MESSAGE_LEN     .req    r2
        HASH            .req    r3

        PASS0_SUMS      .req    q0
        PASS0_SUM_A     .req    d0
        PASS0_SUM_B     .req    d1
        PASS1_SUMS      .req    q1
        PASS1_SUM_A     .req    d2
        PASS1_SUM_B     .req    d3
        PASS2_SUMS      .req    q2
        PASS2_SUM_A     .req    d4
        PASS2_SUM_B     .req    d5
        PASS3_SUMS      .req    q3
        PASS3_SUM_A     .req    d6
        PASS3_SUM_B     .req    d7
        K0              .req    q4
        K1              .req    q5
        K2              .req    q6
        K3              .req    q7
        T0              .req    q8
        T0_L            .req    d16
        T0_H            .req    d17
        T1              .req    q9
        T1_L            .req    d18
        T1_H            .req    d19
        T2              .req    q10
        T2_L            .req    d20
        T2_H            .req    d21
        T3              .req    q11
        T3_L            .req    d22
        T3_H            .req    d23

.macro _nh_stride       k0, k1, k2, k3

        // Load next message stride
        vld1.8          {T3}, [MESSAGE]!

        // Load next key stride
        vld1.32         {\k3}, [KEY]!

        // Add message words to key words
        vadd.u32        T0, T3, \k0
        vadd.u32        T1, T3, \k1
        vadd.u32        T2, T3, \k2
        vadd.u32        T3, T3, \k3

        // Multiply 32x32 => 64 and accumulate
        vmlal.u32       PASS0_SUMS, T0_L, T0_H
        vmlal.u32       PASS1_SUMS, T1_L, T1_H
        vmlal.u32       PASS2_SUMS, T2_L, T2_H
        vmlal.u32       PASS3_SUMS, T3_L, T3_H
.endm

/*
 * void nh_neon(const u32 *key, const u8 *message, size_t message_len,
 *              __le64 hash[NH_NUM_PASSES])
 *
 * It's guaranteed that message_len % 16 == 0.
 */
ENTRY(nh_neon)

        vld1.32         {K0,K1}, [KEY]!
          vmov.u64      PASS0_SUMS, #0
          vmov.u64      PASS1_SUMS, #0
        vld1.32         {K2}, [KEY]!
          vmov.u64      PASS2_SUMS, #0
          vmov.u64      PASS3_SUMS, #0

        subs            MESSAGE_LEN, MESSAGE_LEN, #64
        blt             .Lloop4_done
.Lloop4:
        _nh_stride      K0, K1, K2, K3
        _nh_stride      K1, K2, K3, K0
        _nh_stride      K2, K3, K0, K1
        _nh_stride      K3, K0, K1, K2
        subs            MESSAGE_LEN, MESSAGE_LEN, #64
        bge             .Lloop4

.Lloop4_done:
        ands            MESSAGE_LEN, MESSAGE_LEN, #63
        beq             .Ldone
        _nh_stride      K0, K1, K2, K3

        subs            MESSAGE_LEN, MESSAGE_LEN, #16
        beq             .Ldone
        _nh_stride      K1, K2, K3, K0

        subs            MESSAGE_LEN, MESSAGE_LEN, #16
        beq             .Ldone
        _nh_stride      K2, K3, K0, K1

.Ldone:
        // Sum the accumulators for each pass, then store the sums to 'hash'
        vadd.u64        T0_L, PASS0_SUM_A, PASS0_SUM_B
        vadd.u64        T0_H, PASS1_SUM_A, PASS1_SUM_B
        vadd.u64        T1_L, PASS2_SUM_A, PASS2_SUM_B
        vadd.u64        T1_H, PASS3_SUM_A, PASS3_SUM_B
        vst1.8          {T0-T1}, [HASH]
        bx              lr
ENDPROC(nh_neon)