common/lib/libc/arch/aarch64/atomic/__aarch64_lse.S

root/common/lib/libc/arch/aarch64/atomic/__aarch64_lse.S
/* $NetBSD: __aarch64_lse.S,v 1.7 2022/08/06 21:31:33 riastradh Exp $ */

/*-
 * Copyright (c) 2021 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Nick Hudson.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>

#include "atomic_op_asm.h"

#if SZ == 1
#define OPSFX   b
#define R0      w0
#define R1      w1
#define R4      w4
#endif

#if SZ == 2
#define OPSFX   h
#define R0      w0
#define R1      w1
#define R4      w4
#endif

#if SZ == 4
#define OPSFX
#define R0      w0
#define R1      w1
#define R4      w4
#endif

#if SZ == 8
#define OPSFX
#define R0      x0
#define R1      x1
#define R4      x4
#endif

#if defined(AR_relax)
#define ACQ
#define REL
#define DMB
#endif

#if defined(AR_acq)
#define ACQ     a
#define REL
#define DMB
#endif

#if defined(AR_rel)
#define ACQ
#define REL     l
#define DMB
#endif

#if defined(AR_acq_rel)
#define ACQ     a
#define REL     l
#define DMB
#endif

#if defined(AR_sync)
#define ACQ
#define REL
#define DMB     dmb ish
#endif

#if defined(OP_clr)
#define INSNOP  bic
#endif

#if defined(OP_set)
#define INSNOP  orr
#endif

#if defined(OP_add)
#define INSNOP  add
#endif

#if defined(OP_eor)
#define INSNOP  eor
#endif

#define _CONCAT3(A, B, C)       __CONCAT3(A,B,C)
#define _CONCAT4(A, B, C, D)    __CONCAT4(A,B,C,D)
#define _CONCAT5(A, B, C, D, E) __CONCAT5(A,B,C,D,E)

#define FUNC2                   _CONCAT3(__aarch64_,OP,AR)
#define FUNC3                   _CONCAT4(__aarch64_,OP,SZ,AR)

#define CASP_FUNC               FUNC2
#define CAS_FUNC                FUNC3
#define SWP_FUNC                FUNC3
#define INSN_FUNC               FUNC3

#define LDXR                    _CONCAT4(ld, ACQ, xr, OPSFX)
#define STXR                    _CONCAT4(st, REL, xr, OPSFX)
#define LDXP                    _CONCAT3(ld, ACQ, xp)
#define STXP                    _CONCAT3(st, REL, xp)

#ifdef _HAVE_LSE
#define SWP                     _CONCAT4(swp, ACQ, REL, OPSFX)
#define CAS                     _CONCAT4(cas, ACQ, REL, OPSFX)
#define CASP                    _CONCAT3(casp, ACQ, REL)
#define INSN                    _CONCAT5(ld, OP, ACQ, REL, OPSFX)

        .hidden __aarch64_have_lse_atomics
        .arch armv8-a+lse

#define DO_LSE_INSN_IF_SUPPORTED(label)                         \
        adrp    x4, __aarch64_have_lse_atomics                  ;\
        ldrb    w4, [x4, #:lo12:__aarch64_have_lse_atomics]     ;\
        cbnz    w4, label

#endif

#if defined(OP_swp)
ENTRY_NP(SWP_FUNC)
#ifdef _HAVE_LSE
        DO_LSE_INSN_IF_SUPPORTED(99f)
        DMB
        SWP     R0, R0, [x1]
        DMB
        ret
99:
#endif
        mov     x4, x0                  /* need x0 for return value     */
        DMB                             /* potential barrier            */
1:      LDXR    R0, [x1]                /* load old value               */
        STXR    w3, R4, [x1]            /* store new value              */
        cbnz    w3, 2f                  /*   succeed?? no, try again    */
        DMB                             /* potential barrier            */
        ret                             /* return old value             */
2:      b       1b
END(SWP_FUNC)
#endif

#if defined(OP_cas)
ENTRY_NP(CAS_FUNC)
#ifdef _HAVE_LSE
        DO_LSE_INSN_IF_SUPPORTED(99f)
        DMB
        CAS     R0, R1, [x2]
        DMB
        ret
99:
#endif
        mov     x4, x0                  /* need x0 for return value     */
        DMB                             /* potential barrier            */
1:      LDXR    R0, [x2]                /* load old value               */
        cmp     R0, R4                  /* compare                      */
        b.ne    2f                      /*   not equal? return          */
        STXR    w3, R1, [x2]            /* store new value              */
        cbnz    w3, 3f                  /*   succeed? nope, try again.  */
        DMB                             /* potential barrier            */
2:      ret                             /* return.                      */
3:      b       1b
END(CAS_FUNC)
#endif

#if defined(OP_casp)
ENTRY_NP(CASP_FUNC)
#ifdef _HAVE_LSE
        DO_LSE_INSN_IF_SUPPORTED(99f)
        DMB
        CASP    x0, x1, x2, x3, [x4]
        DMB
        ret
99:
#endif
        mov     x5, x0                  /* need x0 for return value     */
        mov     x6, x1                  /* need x1 for return value     */
        DMB                             /* potential barrier            */
1:      LDXP    x0, x1, [x4]            /* load old value               */
        cmp     x0, x5                  /* compare                      */
        b.ne    2f                      /*   not equal? return          */
        cmp     x1, x6
        b.ne    2f                      /*   not equal? return          */
        STXP    w7, x2, x3, [x4]        /* store new value              */
        cbnz    w7, 3f                  /*   succeed? nope, try again.  */
        DMB                             /* potential barrier            */
2:      ret                             /* return.                      */
3:      b       1b
END(CASP_FUNC)
#endif

#if defined(OP_set) || defined(OP_clr) || defined(OP_add) || defined(OP_eor)
ENTRY_NP(INSN_FUNC)
#ifdef _HAVE_LSE
        DO_LSE_INSN_IF_SUPPORTED(99f)
        DMB
        INSN    R0, R0, [x1]
        DMB
        ret
99:
#endif
        mov     x4, x0                  /* need x0 for return value     */
        DMB                             /* potential barrier            */
1:      LDXR    R0, [x1]                /* load old value               */
        INSNOP  R4, R0, R4
        STXR    w3, R4, [x1]            /* store new value              */
        cbnz    w3, 2f                  /*   succeed?? no, try again    */
        DMB                             /* potential barrier            */
        ret                             /* return old value             */
2:      b       1b
END(INSN_FUNC)
#endif