/*      $NetBSD: lock_stubs_llsc.S,v 1.18 2023/12/05 17:38:40 andvar Exp $      */

/*-
 * Copyright (c) 2007 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Andrew Doran.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include "opt_cputype.h"
#include "opt_lockdebug.h"

#include <sys/errno.h>

#include <machine/asm.h>

RCSID("$NetBSD: lock_stubs_llsc.S,v 1.18 2023/12/05 17:38:40 andvar Exp $")

#include "assym.h"

/*
 * Set ISA level for the assembler.
 * XXX Clean up with a macro?  Same code fragment is in mipsX_subr.S too.
 * XXX Key off build abi instead of processor type?
 */
#if defined(MIPS3)
        .set    mips3
#endif

#if defined(MIPS32)
        .set    mips32
#endif

#if defined(MIPS64)
        .set    mips64
#endif

        .set    noreorder
        .set    noat

/*
 * unsigned long atomic_cas_ulong_llsc(volatile unsigned long *val,
 *     unsigned long old, unsigned long new);
 *
 *      For hysterical raisins in sys/arch/mips/include/lock.h, success
 *      implies load-acquire.  The SYNC_ACQ here could be moved there
 *      instead.
 */
STATIC_LEAF(llsc_atomic_cas_ulong)
        LLSCSYNC
1:
        LONG_LL t0, (a0)
        bne     t0, a1, 2f
         move t1, a2
        LONG_SC t1, (a0)
        beqz    t1, 1b
         nop
        SYNC_ACQ
        j       ra
         move   v0, a1
2:
        j       ra
         move   v0, t0
END(llsc_atomic_cas_ulong)

/*
 * unsigned int _atomic_cas_uint_llsc(volatile unsigned int *val,
 *    unsigned int old, unsigned int new);
 *
 *      For hysterical raisins in sys/arch/mips/include/lock.h, success
 *      implies load-acquire.  The SYNC_ACQ here could be moved there
 *      instead.
 */
STATIC_LEAF(llsc_atomic_cas_uint)
        LLSCSYNC
1:
        INT_LL  t0, (a0)
        bne     t0, a1, 2f
         move   t1, a2
        INT_SC  t1, (a0)
        beqz    t1, 1b
         nop
        SYNC_ACQ
        j       ra
         move   v0, a1
2:
        j       ra
         move   v0, t0
END(llsc_atomic_cas_uint)

/*
 * int llsc_ucas_32(volatile uint32_t *ptr, uint32_t old,
 *      uint32_t new, uint32_t *ret)
 *
 *      Implies release/acquire barriers until someone tells me
 *      otherwise about _ucas_32/64.
 */
STATIC_LEAF(llsc_ucas_32)
        .set at
        PTR_LA  v0, _C_LABEL(llsc_ucaserr)
        .set noat
        PTR_L   v1, L_PCB(MIPS_CURLWP)
        PTR_S   v0, PCB_ONFAULT(v1)
        bltz    a0, _C_LABEL(llsc_ucaserr)
         nop
        move    v0, zero
        SYNC_REL

        LLSCSYNC
1:      ll      t0, 0(a0)
        bne     t0, a1, 2f
         move   t1, a2
        sc      t1, 0(a0)
        beqz    t1, 1b
         nop
        SYNC_ACQ

2:      PTR_S   zero, PCB_ONFAULT(v1)
        j       ra
         sw     t0, 0(a3)
END(llsc_ucas_32)

#ifdef _LP64
/*
 * int llsc_ucas_64(volatile uint64_t *ptr, uint64_t old,
 *      uint64_t new, uint64_t *ret)
 */
STATIC_LEAF(llsc_ucas_64)
        .set at
        PTR_LA  v0, _C_LABEL(llsc_ucaserr)
        .set noat
        PTR_L   v1, L_PCB(MIPS_CURLWP)
        PTR_S   v0, PCB_ONFAULT(v1)
        bltz    a0, _C_LABEL(llsc_ucaserr)
         nop
        move    v0, zero
        SYNC_REL

        LLSCSYNC
1:      lld     t0, 0(a0)
        bne     t0, a1, 2f
         move   t1, a2
        scd     t1, 0(a0)
        beqz    t1, 1b
         nop
        SYNC_ACQ

2:      PTR_S   zero, PCB_ONFAULT(v1)
        j       ra
         sd     t0, 0(a3)
END(llsc_ucas_64)
#endif /* _LP64 */

STATIC_LEAF_NOPROFILE(llsc_ucaserr)
        PTR_S   zero, PCB_ONFAULT(v1)           # reset fault handler
        j       ra
         li     v0, EFAULT                      # return EFAULT on error
END(llsc_ucaserr)

#ifndef LOCKDEBUG

/*
 * void mutex_enter(kmutex_t *mtx);
 */
STATIC_LEAF(llsc_mutex_enter)
        LLSCSYNC
        PTR_LL  t0, MTX_OWNER(a0)
1:
        bnez    t0, 2f
         move   t2, MIPS_CURLWP
        PTR_SC  t2, MTX_OWNER(a0)
        beqz    t2, 1b
         PTR_LL t0, MTX_OWNER(a0)
        j       ra
         BDSYNC_ACQ
2:
        j       _C_LABEL(mutex_vector_enter)
         nop
END(llsc_mutex_enter)

/*
 * void mutex_exit(kmutex_t *mtx);
 */
STATIC_LEAF(llsc_mutex_exit)
        SYNC_REL
        LLSCSYNC
        PTR_LL  t0, MTX_OWNER(a0)
1:
        bne     t0, MIPS_CURLWP, 2f
         move   t2, zero
        PTR_SC  t2, MTX_OWNER(a0)
        beqz    t2, 1b
         PTR_LL t0, MTX_OWNER(a0)
        j       ra
         BDSYNC_PLUNGER
2:
        j       _C_LABEL(mutex_vector_exit)
         nop
END(llsc_mutex_exit)

/*
 * void mutex_spin_enter(kmutex_t *mtx);
 */
STATIC_NESTED(llsc_mutex_spin_enter, CALLFRAME_SIZ, ra)
        move    t0, a0
        PTR_L   t2, L_CPU(MIPS_CURLWP)
        INT_L   a0, MTX_IPL(t0)
#ifdef PARANOIA
        INT_L   ta1, CPU_INFO_CPL(t2)
#endif

        /*
         * We need to raise our IPL.  But it means calling another routine
         * but it's written to have little overhead.  call splraise
         * (only uses a0-a3 and v0-v1)
         */
        move    t3, ra                  # need to save ra
        jal     _C_LABEL(splraise)
         nop
        move    ra, t3                  # move ra back
#ifdef PARANOIA
10:     bne     ta1, v0, 10b            # loop forever if v0 != ta1
         nop
#endif /* PARANOIA */

        /*
         * If this is the first lock of the mutex, store the previous IPL for
         * exit.  Even if an interrupt happens, the mutex count will not change.
         */
1:
        INT_L   ta2, CPU_INFO_MTX_COUNT(t2)
        INT_ADDU ta3, ta2, -1
        INT_S   ta3, CPU_INFO_MTX_COUNT(t2)
        bltz    ta2, 2f
         nop
        INT_S   v0, CPU_INFO_MTX_OLDSPL(t2)     /* returned by splraise */
2:
#ifdef PARANOIA
        INT_L   ta1, CPU_INFO_MTX_OLDSPL(t2)
        INT_L   ta2, CPU_INFO_CPL(t2)   # get updated CPL
        sltu    v0, ta2, ta0            # v0 = cpl < mtx_ipl
        sltu    v1, ta2, ta1            # v1 = cpl < oldspl
        sll     v0, 1
        or      v0, v1
12:     bnez    v0, 12b                 # loop forever if any are true
         nop
#endif /* PARANOIA */

        LLSCSYNC
        INT_LL  t3, MTX_LOCK(t0)
3:
        bnez    t3, 4f
         li     t1, 1
        INT_SC  t1, MTX_LOCK(t0)
        beqz    t1, 3b
         INT_LL t3, MTX_LOCK(t0)
        j       ra
         BDSYNC_ACQ
4:
        j       _C_LABEL(mutex_spin_retry)
         move   a0, t0
END(llsc_mutex_spin_enter)

/*
 * void mutex_spin_exit(kmutex_t *mtx);
 */
LEAF(llsc_mutex_spin_exit)
        SYNC_REL
        PTR_L   t2, L_CPU(MIPS_CURLWP)
#if defined(DIAGNOSTIC)
        INT_L   t0, MTX_LOCK(a0)
        beqz    t0, 2f
         nop
#endif
        INT_S   zero, MTX_LOCK(a0)

        /*
         * We need to grab this before the mutex count is incremented
         * because if we get an interrupt, it may see the count as zero
         * and overwrite the oldspl value with a bogus value.
         */
#ifdef PARANOIA
        INT_L   a2, MTX_IPL(a0)
#endif
        INT_L   a0, CPU_INFO_MTX_OLDSPL(t2)

        /*
         * Increment the mutex count
         */
        INT_L   t0, CPU_INFO_MTX_COUNT(t2)
        INT_ADDU t0, t0, 1
        INT_S   t0, CPU_INFO_MTX_COUNT(t2)

        /*
         * If the IPL doesn't change, nothing to do
         */
        INT_L   a1, CPU_INFO_CPL(t2)

#ifdef PARANOIA
        sltu    v0, a1, a2              # v0 = cpl < mtx_ipl
        sltu    v1, a1, a0              # v1 = cpl < oldspl
        sll     v0, 1
        or      v0, v1
12:     bnez    v0, 12b                 # loop forever if either is true
         nop
#endif /* PARANOIA */

        beq     a0, a1, 1f              # if oldspl == cpl
         nop                            #   no reason to drop ipl

        bltz    t0, 1f                  # there are still holders
         nop                            # so don't drop IPL

        /*
         * Mutex count is zero so we need to restore the old IPL
         */
#ifdef PARANOIA
        sltiu   v0, a0, IPL_HIGH+1
13:     beqz    v0, 13b                 # loop forever if ipl > IPL_HIGH
         nop
#endif
        j        _C_LABEL(splx)
         BDSYNC_PLUNGER
1:
        j       ra
         BDSYNC_PLUNGER
#if defined(DIAGNOSTIC)
2:
        j       _C_LABEL(mutex_vector_exit)
         nop
#endif
END(llsc_mutex_spin_exit)
#endif  /* !LOCKDEBUG */

        .rdata
EXPORT_OBJECT(mips_llsc_locore_atomicvec)
        PTR_WORD        llsc_atomic_cas_uint
        PTR_WORD        llsc_atomic_cas_ulong
        PTR_WORD        llsc_ucas_32
#ifdef _LP64
        PTR_WORD        llsc_ucas_64
#else
        PTR_WORD        0
#endif /* _LP64 */
#ifdef LOCKDEBUG
        PTR_WORD        mutex_vector_enter
        PTR_WORD        mutex_vector_exit
        PTR_WORD        mutex_vector_enter
        PTR_WORD        mutex_vector_exit
#else
        PTR_WORD        llsc_mutex_enter
        PTR_WORD        llsc_mutex_exit
        PTR_WORD        llsc_mutex_spin_enter
        PTR_WORD        llsc_mutex_spin_exit
#endif  /* !LOCKDEBUG */