root/sys/crypto/arch/arm/arm_neon.h 
/*      $NetBSD: arm_neon.h,v 1.2 2023/08/07 01:14:19 rin Exp $ */

/*-
 * Copyright (c) 2020 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#ifndef _SYS_CRYPTO_ARCH_ARM_ARM_NEON_H
#define _SYS_CRYPTO_ARCH_ARM_ARM_NEON_H

#if defined(__GNUC__) && !defined(__clang__)

#define _INTRINSATTR                                                          \
        __extension__                                                         \
        __attribute__((__always_inline__, __gnu_inline__, __artificial__))

#ifdef __aarch64__
typedef __Int32x4_t int32x4_t;
typedef __Int64x2_t int64x2_t;
typedef __Int8x16_t int8x16_t;
typedef __Uint16x8_t uint16x8_t;
typedef __Uint32x4_t uint32x4_t;
typedef __Uint64x2_t uint64x2_t;
typedef __Uint8x16_t uint8x16_t;
typedef struct { uint8x16_t val[2]; } uint8x16x2_t;
#else
typedef __simd128_int32_t int32x4_t;
typedef __simd128_int64_t int64x2_t;
typedef __simd128_int8_t int8x16_t;
typedef __simd128_uint16_t uint16x8_t;
typedef __simd128_uint32_t uint32x4_t;
typedef __simd128_uint64_t uint64x2_t;
typedef __simd128_uint8_t uint8x16_t;

typedef __simd64_int8_t int8x8_t;
typedef __simd64_uint8_t uint8x8_t;
typedef __builtin_neon_udi uint64x1_t;
typedef struct { uint8x8_t val[2]; } uint8x8x2_t;
typedef struct { uint8x16_t val[2]; } uint8x16x2_t;
#endif

#if defined(__AARCH64EB__)
#define __neon_lane_index(__v, __i)     (__arraycount(__v) - 1 - (__i))
#define __neon_laneq_index(__v, __i)    (__arraycount(__v) - 1 - (__i))
#elif defined(__ARM_BIG_ENDIAN)
#define __neon_lane_index(__v, __i)     ((__i) ^ (__arraycount(__v) - 1))
#define __neon_laneq_index(__v, __i)    ((__i) ^ (__arraycount(__v)/2 - 1))
#else
#define __neon_lane_index(__v, __i)     (__i)
#define __neon_laneq_index(__v, __i)    (__i)
#endif

#elif defined(__clang__)

#define _INTRINSATTR                                                          \
        __attribute__((__always_inline__, __nodebug__))

typedef __attribute__((neon_vector_type(16))) int8_t int8x16_t;
typedef __attribute__((neon_vector_type(2))) int64_t int64x2_t;
typedef __attribute__((neon_vector_type(4))) int32_t int32x4_t;

typedef __attribute__((neon_vector_type(16))) uint8_t uint8x16_t;
typedef __attribute__((neon_vector_type(2))) uint64_t uint64x2_t;
typedef __attribute__((neon_vector_type(4))) uint32_t uint32x4_t;
typedef __attribute__((neon_vector_type(8))) uint16_t uint16x8_t;

typedef __attribute__((neon_vector_type(8))) int8_t int8x8_t;

typedef __attribute__((neon_vector_type(8))) uint8_t uint8x8_t;

typedef struct { uint8x8_t val[2]; } uint8x8x2_t;
typedef struct { uint8x16_t val[2]; } uint8x16x2_t;

#ifdef __LITTLE_ENDIAN__
#define __neon_lane_index(__v, __i)     __i
#define __neon_laneq_index(__v, __i)    __i
#else
#define __neon_lane_index(__v, __i)     (__arraycount(__v) - 1 - __i)
#define __neon_laneq_index(__v, __i)    (__arraycount(__v) - 1 - __i)
#endif

#else

#error Teach me how to neon in your compile!

#endif

_INTRINSATTR
static __inline uint32x4_t
vaddq_u32(uint32x4_t __v0, uint32x4_t __v1)
{
        return __v0 + __v1;
}

_INTRINSATTR
static __inline uint32x4_t
vcltq_s32(int32x4_t __v0, int32x4_t __v1)
{
        return (uint32x4_t)(__v0 < __v1);
}

_INTRINSATTR
static __inline int32x4_t
vdupq_n_s32(int32_t __x)
{
        return (int32x4_t) { __x, __x, __x, __x };
}

_INTRINSATTR
static __inline uint32x4_t
vdupq_n_u32(uint32_t __x)
{
        return (uint32x4_t) { __x, __x, __x, __x };
}

_INTRINSATTR
static __inline uint8x16_t
vdupq_n_u8(uint8_t __x)
{
        return (uint8x16_t) {
                __x, __x, __x, __x, __x, __x, __x, __x,
                __x, __x, __x, __x, __x, __x, __x, __x,
        };
}

#if defined(__GNUC__) && !defined(__clang__)
_INTRINSATTR
static __inline uint32x4_t
vextq_u32(uint32x4_t __lo, uint32x4_t __hi, uint8_t __i)
{
#if defined(__AARCH64EB__) || defined(__ARM_BIG_ENDIAN)
        return __builtin_shuffle(__hi, __lo,
            (uint32x4_t) { 4 - __i, 5 - __i, 6 - __i, 7 - __i });
#else
        return __builtin_shuffle(__lo, __hi,
            (uint32x4_t) { __i + 0, __i + 1, __i + 2, __i + 3 });
#endif
}
#elif defined(__clang__)
#ifdef __LITTLE_ENDIAN__
#define vextq_u32(__lo, __hi, __i)                                            \
        (uint32x4_t)__builtin_neon_vextq_v((int8x16_t)(__lo),                 \
            (int8x16_t)(__hi), (__i), 50)
#else
#define vextq_u32(__lo, __hi, __i) (                                          \
{                                                                             \
        uint32x4_t __tlo = (__lo);                                            \
        uint32x4_t __thi = (__hi);                                            \
        uint32x4_t __lo_r = __builtin_shufflevector(__tlo, __tlo, 3,2,1,0);   \
        uint32x4_t __hi_r = __builtin_shufflevector(__thi, __thi, 3,2,1,0);   \
        uint32x4_t __r = __builtin_neon_vextq_v((int8x16_t)__lo_r,            \
            (int8x16_t)__hi_r, __i, 50);                                      \
        __builtin_shufflevector(__r, __r, 3,2,1,0);                           \
})
#endif  /* __LITTLE_ENDIAN__ */
#endif

#if defined(__GNUC__) && !defined(__clang__)
_INTRINSATTR
static __inline uint8x16_t
vextq_u8(uint8x16_t __lo, uint8x16_t __hi, uint8_t __i)
{
#ifdef __aarch64__
#if defined(__AARCH64EB__)
        return __builtin_shuffle(__hi, __lo,
            (uint8x16_t) {
                16 - __i, 17 - __i, 18 - __i, 19 - __i,
                20 - __i, 21 - __i, 22 - __i, 23 - __i,
                24 - __i, 25 - __i, 26 - __i, 27 - __i,
                28 - __i, 29 - __i, 30 - __i, 31 - __i,
        });
#else
        return __builtin_shuffle(__lo, __hi,
            (uint8x16_t) {
                __i +  0, __i +  1, __i +  2, __i +  3,
                __i +  4, __i +  5, __i +  6, __i +  7,
                __i +  8, __i +  9, __i + 10, __i + 11,
                __i + 12, __i + 13, __i + 14, __i + 15,
        });
#endif
#else
        return (uint8x16_t)__builtin_neon_vextv16qi((int8x16_t)__lo,
            (int8x16_t)__hi, __i);
#endif
}
#elif defined(__clang__)
#ifdef __LITTLE_ENDIAN__
#define vextq_u8(__lo, __hi, __i)                                             \
        (uint8x16_t)__builtin_neon_vextq_v((int8x16_t)(__lo),                 \
            (int8x16_t)(__hi), (__i), 48)
#else
#define vextq_u8(__lo, __hi, __i) (                                           \
{                                                                             \
        uint8x16_t __tlo = (__lo);                                            \
        uint8x16_t __thi = (__hi);                                            \
        uint8x16_t __lo_r = __builtin_shufflevector(__tlo, __tlo,             \
            15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0);                           \
        uint8x16_t __hi_r = __builtin_shufflevector(__thi, __thi,             \
            15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0);                           \
        uint8x16_t __r = __builtin_neon_vextq_v((int8x16_t)__lo_r,            \
            (int8x16_t)__hi_r, (__i), 48);                                    \
        __builtin_shufflevector(__r, __r,                                     \
            15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0);                           \
})
#endif  /* __LITTLE_ENDIAN */
#endif

#if defined(__GNUC__) && !defined(__clang__)
_INTRINSATTR
static __inline uint32_t
vgetq_lane_u32(uint32x4_t __v, uint8_t __i)
{
#ifdef __aarch64__
        return __v[__neon_laneq_index(__v, __i)];
#else
        return (uint32_t)__builtin_neon_vget_laneuv4si((int32x4_t)__v, __i);
#endif
}
#elif defined(__clang__)
#define vgetq_lane_u32(__v, __i)                                              \
        (uint32_t)__builtin_neon_vgetq_lane_i32((int32x4_t)(__v),             \
            __neon_laneq_index(__v, __i))
#endif

_INTRINSATTR
static __inline uint32x4_t
vld1q_u32(const uint32_t *__p32)
{
#if defined(__GNUC__) && !defined(__clang__)
#ifdef __aarch64__
        const __builtin_aarch64_simd_si *__p =
            (const __builtin_aarch64_simd_si *)__p32;

        return (uint32x4_t)__builtin_aarch64_ld1v4si(__p);
#else
        const __builtin_neon_si *__p = (const __builtin_neon_si *)__p32;

        return (uint32x4_t)__builtin_neon_vld1v4si(__p);
#endif
#elif defined(__clang__)
        uint32x4_t __v = (uint32x4_t)__builtin_neon_vld1q_v(__p32, 50);
#ifndef __LITTLE_ENDIAN__
        __v = __builtin_shufflevector(__v, __v, 3,2,1,0);
#endif
        return __v;
#endif
}

_INTRINSATTR
static __inline uint8x16_t
vld1q_u8(const uint8_t *__p8)
{
#if defined(__GNUC__) && !defined(__clang__)
#ifdef __aarch64__
        const __builtin_aarch64_simd_qi *__p =
            (const __builtin_aarch64_simd_qi *)__p8;

        return (uint8x16_t)__builtin_aarch64_ld1v16qi(__p);
#else
        const __builtin_neon_qi *__p = (const __builtin_neon_qi *)__p8;

        return (uint8x16_t)__builtin_neon_vld1v16qi(__p);
#endif
#elif defined(__clang__)
        uint8x16_t __v = (uint8x16_t)__builtin_neon_vld1q_v(__p8, 48);
#ifndef __LITTLE_ENDIAN__
        __v = __builtin_shufflevector(__v, __v,
            15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0);
#endif
        return __v;
#endif
}

_INTRINSATTR
static __inline uint8x16_t
vqtbl1q_u8(uint8x16_t __tab, uint8x16_t __idx)
{
#if defined(__GNUC__) && !defined(__clang__)
#ifdef __aarch64__
        uint8x16_t __res;
        __asm__("tbl %0.16b, {%1.16b}, %2.16b"
            : "=w"(__res) : "w"(__tab), "w"(__idx));
        return __res;
#else
        /*
         * No native ARMv7 NEON instruction for this, so do it via two
         * half-width TBLs instead (vtbl2_u8 equivalent).
         */
        uint64x2_t __tab64 = (uint64x2_t)__tab;
        uint8x8_t __tablo = (uint8x8_t)__tab64[0];
        uint8x8_t __tabhi = (uint8x8_t)__tab64[1];
        uint8x8x2_t __tab8x8x2 = { { __tablo, __tabhi } };
        union {
                uint8x8x2_t __u8x8x2;
                __builtin_neon_ti __ti;
        } __u = { __tab8x8x2 };
        uint64x2_t __idx64, __out64;
        int8x8_t __idxlo, __idxhi, __outlo, __outhi;

        __idx64 = (uint64x2_t)__idx;
        __idxlo = (int8x8_t)__idx64[0];
        __idxhi = (int8x8_t)__idx64[1];
        __outlo = (int8x8_t)__builtin_neon_vtbl2v8qi(__u.__ti, __idxlo);
        __outhi = (int8x8_t)__builtin_neon_vtbl2v8qi(__u.__ti, __idxhi);
        __out64 = (uint64x2_t) { (uint64x1_t)__outlo, (uint64x1_t)__outhi };

        return (uint8x16_t)__out64;
#endif
#elif defined(__clang__)
#ifndef __LITTLE_ENDIAN__
        __tab = __builtin_shufflevector(__tab, __tab,
            15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0);
        __idx = __builtin_shufflevector(__idx, __idx,
            15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0);
#endif
        uint8x16_t __r;
#ifdef __aarch64__
        __r = __builtin_neon_vqtbl1q_v((int8x16_t)__tab, (int8x16_t)__idx, 48);
#else
        uint64x2_t __tab64 = (uint64x2_t)__tab;
        uint8x8_t __tablo = (uint8x8_t)__tab64[0];
        uint8x8_t __tabhi = (uint8x8_t)__tab64[1];
        uint64x2_t __idx64, __out64;
        int8x8_t __idxlo, __idxhi, __outlo, __outhi;

        __idx64 = (uint64x2_t)__idx;
        __idxlo = (int8x8_t)__idx64[0];
        __idxhi = (int8x8_t)__idx64[1];
        __outlo = (uint8x8_t)__builtin_neon_vtbl2_v((int8x8_t)__tablo,
            (int8x8_t)__tabhi, (int8x8_t)__idxlo, 16);
        __outhi = (uint8x8_t)__builtin_neon_vtbl2_v((int8x8_t)__tablo,
            (int8x8_t)__tabhi, (int8x8_t)__idxhi, 16);
        __out64 = (uint64x2_t) { (uint64_t)__outlo, (uint64_t)__outhi };
        __r = (uint8x16_t)__out64;
#endif
#ifndef __LITTLE_ENDIAN__
        __r = __builtin_shufflevector(__r, __r,
            15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0);
#endif
        return __r;
#endif
}

_INTRINSATTR
static __inline int32x4_t
vreinterpretq_s32_u8(uint8x16_t __v)
{
        return (int32x4_t)__v;
}

_INTRINSATTR
static __inline uint16x8_t
vreinterpretq_u16_u32(uint32x4_t __v)
{
        return (uint16x8_t)__v;
}

_INTRINSATTR
static __inline uint32x4_t
vreinterpretq_u32_u16(uint16x8_t __v)
{
        return (uint32x4_t)__v;
}

_INTRINSATTR
static __inline uint32x4_t
vreinterpretq_u32_u64(uint64x2_t __v)
{
        return (uint32x4_t)__v;
}

_INTRINSATTR
static __inline uint32x4_t
vreinterpretq_u32_u8(uint8x16_t __v)
{
        return (uint32x4_t)__v;
}

_INTRINSATTR
static __inline uint64x2_t
vreinterpretq_u64_u32(uint32x4_t __v)
{
        return (uint64x2_t)__v;
}

_INTRINSATTR
static __inline uint64x2_t
vreinterpretq_u64_u8(uint8x16_t __v)
{
        return (uint64x2_t)__v;
}

_INTRINSATTR
static __inline uint8x16_t
vreinterpretq_u8_s32(int32x4_t __v)
{
        return (uint8x16_t)__v;
}

_INTRINSATTR
static __inline uint8x16_t
vreinterpretq_u8_u32(uint32x4_t __v)
{
        return (uint8x16_t)__v;
}

_INTRINSATTR
static __inline uint8x16_t
vreinterpretq_u8_u64(uint64x2_t __v)
{
        return (uint8x16_t)__v;
}

_INTRINSATTR
static __inline uint16x8_t
vrev32q_u16(uint16x8_t __v)
{
#if defined(__GNUC__) && !defined(__clang__)
        return __builtin_shuffle(__v, (uint16x8_t) { 1,0, 3,2, 5,4, 7,6 });
#elif defined(__clang__)
        return __builtin_shufflevector(__v, __v,  1,0, 3,2, 5,4, 7,6);
#endif
}

_INTRINSATTR
static __inline uint8x16_t
vrev32q_u8(uint8x16_t __v)
{
#if defined(__GNUC__) && !defined(__clang__)
        return __builtin_shuffle(__v,
            (uint8x16_t) { 3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12 });
#elif defined(__clang__)
        return __builtin_shufflevector(__v, __v,
            3,2,1,0, 7,6,5,4, 11,10,9,8, 15,14,13,12);
#endif
}

#if defined(__GNUC__) && !defined(__clang__)
_INTRINSATTR
static __inline uint32x4_t
vsetq_lane_u32(uint32_t __x, uint32x4_t __v, uint8_t __i)
{
        __v[__neon_laneq_index(__v, __i)] = __x;
        return __v;
}
#elif defined(__clang__)
#define vsetq_lane_u32(__x, __v, __i)                                         \
        (uint32x4_t)__builtin_neon_vsetq_lane_i32((__x), (int32x4_t)(__v),    \
            __neon_laneq_index(__v, __i))
#endif

#if defined(__GNUC__) && !defined(__clang__)
_INTRINSATTR
static __inline uint64x2_t
vsetq_lane_u64(uint64_t __x, uint64x2_t __v, uint8_t __i)
{
        __v[__neon_laneq_index(__v, __i)] = __x;
        return __v;
}
#elif defined(__clang__)
#define vsetq_lane_u64(__x, __v, __i)                                         \
        (uint64x2_t)__builtin_neon_vsetq_lane_i64((__x), (int64x2_t)(__v),    \
            __neon_laneq_index(__v, __i));
#endif

#if defined(__GNUC__) && !defined(__clang__)
_INTRINSATTR
static __inline int32x4_t
vshlq_n_s32(int32x4_t __v, uint8_t __bits)
{
#ifdef __aarch64__
        return (int32x4_t)__builtin_aarch64_ashlv4si(__v, __bits);
#else
        return (int32x4_t)__builtin_neon_vshl_nv4si(__v, __bits);
#endif
}
#elif defined(__clang__)
#define vshlq_n_s32(__v, __bits)                                              \
        (int32x4_t)__builtin_neon_vshlq_n_v((int32x4_t)(__v), (__bits), 34)
#endif

#if defined(__GNUC__) && !defined(__clang__)
_INTRINSATTR
static __inline uint32x4_t
vshlq_n_u32(uint32x4_t __v, uint8_t __bits)
{
#ifdef __aarch64__
        return (uint32x4_t)__builtin_aarch64_ashlv4si((int32x4_t)__v, __bits);
#else
        return (uint32x4_t)__builtin_neon_vshl_nv4si((int32x4_t)__v, __bits);
#endif
}
#elif defined(__clang__)
#define vshlq_n_u32(__v, __bits)                                              \
        (uint32x4_t)__builtin_neon_vshlq_n_v((int32x4_t)(__v), (__bits), 50)
#endif

#if defined(__GNUC__) && !defined(__clang__)
_INTRINSATTR
static __inline uint32x4_t
vshrq_n_u32(uint32x4_t __v, uint8_t __bits)
{
#ifdef __aarch64__
#  if __GNUC_PREREQ__(12, 0)
        return __builtin_aarch64_lshrv4si_uus(__v, __bits);
#  else
        return (uint32x4_t)__builtin_aarch64_lshrv4si((int32x4_t)__v, __bits);
#  endif
#else
        return (uint32x4_t)__builtin_neon_vshru_nv4si((int32x4_t)__v, __bits);
#endif
}
#elif defined(__clang__)
#define vshrq_n_u32(__v, __bits)                                              \
        (uint32x4_t)__builtin_neon_vshrq_n_v((int32x4_t)(__v), (__bits), 50)
#endif

#if defined(__GNUC__) && !defined(__clang__)
_INTRINSATTR
static __inline uint8x16_t
vshrq_n_u8(uint8x16_t __v, uint8_t __bits)
{
#ifdef __aarch64__
#  if __GNUC_PREREQ__(12, 0)
        return __builtin_aarch64_lshrv16qi_uus(__v, __bits);
#  else
        return (uint8x16_t)__builtin_aarch64_lshrv16qi((int8x16_t)__v, __bits);
#  endif
#else
        return (uint8x16_t)__builtin_neon_vshru_nv16qi((int8x16_t)__v, __bits);
#endif
}
#elif defined(__clang__)
#define vshrq_n_u8(__v, __bits)                                               \
        (uint8x16_t)__builtin_neon_vshrq_n_v((int8x16_t)(__v), (__bits), 48)
#endif

#if defined(__GNUC__) && !defined(__clang__)
_INTRINSATTR
static __inline int32x4_t
vsliq_n_s32(int32x4_t __vins, int32x4_t __vsh, uint8_t __bits)
{
#ifdef __aarch64__
        return (int32x4_t)__builtin_aarch64_ssli_nv4si(__vins, __vsh, __bits);
#else
        return (int32x4_t)__builtin_neon_vsli_nv4si(__vins, __vsh, __bits);
#endif
}
#elif defined(__clang__)
#ifdef __LITTLE_ENDIAN__
#define vsliq_n_s32(__vins, __vsh, __bits)                                    \
        (int32x4_t)__builtin_neon_vsliq_n_v((int32x4_t)(__vins),              \
            (int32x4_t)(__vsh), (__bits), 34)
#else
#define vsliq_n_s32(__vins, __vsh, __bits) (                                  \
{                                                                             \
        int32x4_t __tvins = (__vins);                                         \
        int32x4_t __tvsh = (__vsh);                                           \
        uint8_t __tbits = (__bits);                                           \
        int32x4_t __vins_r = __builtin_shufflevector(__tvins, __tvins,        \
            3,2,1,0);                                                         \
        int32x4_t __vsh_r = __builtin_shufflevector(__tvsh, __tvsh,           \
            3,2,1,0);                                                         \
        int32x4_t __r = __builtin_neon_vsliq_n_v(__tvins, __tvsh, __tbits,    \
            34);                                                              \
        __builtin_shufflevector(__r, __r, 3,2,1,0);                           \
})
#endif  /* __LITTLE_ENDIAN__ */
#endif

#if defined(__GNUC__) && !defined(__clang__)
_INTRINSATTR
static __inline uint32x4_t
vsriq_n_u32(uint32x4_t __vins, uint32x4_t __vsh, uint8_t __bits)
{
#ifdef __aarch64__
        return __builtin_aarch64_usri_nv4si_uuus(__vins, __vsh, __bits);
#else
        return (uint32x4_t)__builtin_neon_vsri_nv4si((int32x4_t)__vins,
            (int32x4_t)__vsh, __bits);
#endif
}
#elif defined(__clang__)
#ifdef __LITTLE_ENDIAN__
#define vsriq_n_u32(__vins, __vsh, __bits)                                    \
        (int32x4_t)__builtin_neon_vsriq_n_v((int32x4_t)(__vins),              \
            (int32x4_t)(__vsh), (__bits), 34)
#else
#define vsriq_n_s32(__vins, __vsh, __bits) (                                  \
{                                                                             \
        int32x4_t __tvins = (__vins);                                         \
        int32x4_t __tvsh = (__vsh);                                           \
        uint8_t __tbits = (__bits);                                           \
        int32x4_t __vins_r = __builtin_shufflevector(__tvins, __tvins,        \
            3,2,1,0);                                                         \
        int32x4_t __vsh_r = __builtin_shufflevector(__tvsh, __tvsh,           \
            3,2,1,0);                                                         \
        int32x4_t __r = __builtin_neon_vsriq_n_v(__tvins, __tvsh, __tbits,    \
            34);                                                              \
        __builtin_shufflevector(__r, __r, 3,2,1,0);                           \
})
#endif
#endif

_INTRINSATTR
static __inline void
vst1q_u32(uint32_t *__p32, uint32x4_t __v)
{
#if defined(__GNUC__) && !defined(__clang__)
#ifdef __aarch64__
        __builtin_aarch64_simd_si *__p = (__builtin_aarch64_simd_si *)__p32;

        __builtin_aarch64_st1v4si(__p, (int32x4_t)__v);
#else
        __builtin_neon_si *__p = (__builtin_neon_si *)__p32;

        __builtin_neon_vst1v4si(__p, (int32x4_t)__v);
#endif
#elif defined(__clang__)
#ifndef __LITTLE_ENDIAN__
        __v = __builtin_shufflevector(__v, __v, 3,2,1,0);
#endif
        __builtin_neon_vst1q_v(__p32, __v, 50);
#endif
}

_INTRINSATTR
static __inline void
vst1q_u8(uint8_t *__p8, uint8x16_t __v)
{
#if defined(__GNUC__) && !defined(__clang__)
#ifdef __aarch64__
        __builtin_aarch64_simd_qi *__p = (__builtin_aarch64_simd_qi *)__p8;

        __builtin_aarch64_st1v16qi(__p, (int8x16_t)__v);
#else
        __builtin_neon_qi *__p = (__builtin_neon_qi *)__p8;

        __builtin_neon_vst1v16qi(__p, (int8x16_t)__v);
#endif
#elif defined(__clang__)
#ifndef __LITTLE_ENDIAN__
        __v = __builtin_shufflevector(__v, __v,
            15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0);
#endif
        __builtin_neon_vst1q_v(__p8, __v, 48);
#endif
}

#ifndef __aarch64__             /* XXX */

_INTRINSATTR
static __inline uint8x8_t
vtbl1_u8(uint8x8_t __tab, uint8x8_t __idx)
{
#if defined(__GNUC__) && !defined(__clang__)
        return (uint8x8_t)__builtin_neon_vtbl1v8qi((int8x8_t)__tab,
            (int8x8_t)__idx);
#elif defined(__clang__)
        uint8x8_t __ret;
#ifndef __LITTLE_ENDIAN__
        __tab = __builtin_shufflevector(__tab, __tab, 7,6,5,4,3,2,1,0);
        __idx = __builtin_shufflevector(__idx, __idx, 7,6,5,4,3,2,1,0);
#endif
        __ret = (uint8x8_t)__builtin_neon_vtbl1_v((int8x8_t)__tab,
            (int8x8_t)__idx, 16);
#ifndef __LITTLE_ENDIAN__
        __ret = __builtin_shufflevector(__ret, __ret, 7,6,5,4,3,2,1,0);
#endif
        return __ret;
#endif
}

_INTRINSATTR
static __inline uint8x8_t
vtbl2_u8(uint8x8x2_t __tab, uint8x8_t __idx)
{
#if defined(__GNUC__) && !defined(__clang__)
        union {
                uint8x8x2_t __u8x8x82;
                __builtin_neon_ti __ti;
        } __u = { __tab };
        return (uint8x8_t)__builtin_neon_vtbl2v8qi(__u.__ti, (int8x8_t)__idx);
#elif defined(__clang__)
        uint8x8_t __ret;
#ifndef __LITTLE_ENDIAN__
        __tab.val[0] = __builtin_shufflevector(__tab.val[0], __tab.val[0],
            7,6,5,4,3,2,1,0);
        __tab.val[1] = __builtin_shufflevector(__tab.val[1], __tab.val[1],
            7,6,5,4,3,2,1,0);
        __idx = __builtin_shufflevector(__idx, __idx, 7,6,5,4,3,2,1,0);
#endif
        __ret = (uint8x8_t)__builtin_neon_vtbl2_v((int8x8_t)__tab.val[0],
            (int8x8_t)__tab.val[1], (int8x8_t)__idx, 16);
#ifndef __LITTLE_ENDIAN__
        __ret = __builtin_shufflevector(__ret, __ret, 7,6,5,4,3,2,1,0);
#endif
        return __ret;
#endif
}

#endif  /* !defined(__aarch64__) */

#endif  /* _SYS_CRYPTO_ARCH_ARM_ARM_NEON_H */