/* $NetBSD: lock_stubs_ras.S,v 1.13 2026/07/02 12:47:52 rkujawa Exp $ */ /*- * Copyright (c) 2007 The NetBSD Foundation, Inc. * All rights reserved. * * This code is derived from software contributed to The NetBSD Foundation * by Andrew Doran. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE * POSSIBILITY OF SUCH DAMAGE. */ #include "opt_cputype.h" #include "opt_lockdebug.h" #include "opt_multiprocessor.h" #include <sys/errno.h> #include <machine/asm.h> RCSID("$NetBSD: lock_stubs_ras.S,v 1.13 2026/07/02 12:47:52 rkujawa Exp $") #include "assym.h" /* * We rely on mips_vector_init to choose to not use these routines if we are * on a system with multiple CPUs. We can still use this on a simple CPU * with MULTIPROCESSOR since it might be useful to be using preemption. */ /* * Lock stubs for non-MP kernels. These are implemented using restartable * sequences, since LL/SC are either not available (MIPS1 and a couple of * oddball MIPS3 CPUs) or not desirable (overhead). * * The order of the generated code is particularly important here. Some * assumptions: * * o All of the critical sections are 20 bytes in size, and the second * instruction in each critical section is aligned on a 16 byte boundary * (see top of _restart_lock_ras() for why). The entry is defined here as * the point where a restart occurs if we trap within the section. * * o The entire code block is aligned on a 256 byte boundary, and is * 256 bytes in size. This is to allow us to do a pessimistic check * after taking a trap with: * * if ((addr & ~255) == _lock_ras_start) * addr = _restart_lock_ras(addr); * * See definition of MIPS_LOCK_RAS_SIZE in asm.h. * * o In order to keep the size of the block down, the routines are run * into each other. Use objdump -d to check alignment after making * changes. */ #ifndef __mips_o32 .set mips3 #else .set mips1 #endif .set noreorder .set noat /* * to work around the branch prediction engine misbehavior of * Loongson 2F processors we need to clear the branch target buffer before * a j ra. This requires extra instructions which don't fit in the RAS blocks, * so do a PC-relative just to a block of code (this is the same size as * a j ra) where we can let the assembler install the workaround. */ #ifdef MIPS3_LOONGSON2F #define J_RA j loongson_return #else #define J_RA j ra #endif /* * unsigned long ras_atomic_cas_ulong(volatile unsigned long *val, * unsigned long old, unsigned long new); */ .text .p2align LOG2_MIPS_LOCK_RAS_SIZE EXPORT(_lock_ras_start) STATIC_LEAF_NOPROFILE(ras_atomic_cas_noupdate) J_RA move v0, t0 END(ras_atomic_cas_noupdate) nop .if ((. - _lock_ras_start) & 15) != 12 .error "bas ras offset" .endif STATIC_LEAF_NOPROFILE(ras_atomic_cas_ulong) PTR_L t0, (a0) /* <- critical section start */ _atomic_cas_ulong_ras_start: nop bne t0, a1, ras_atomic_cas_noupdate nop PTR_S a2, (a0) /* <- critical section end */ _atomic_cas_ulong_ras_end: J_RA move v0, a1 END(ras_atomic_cas_ulong) /* * unsigned int ras_atomic_cas_uint(volatile unsigned int *val, * unsigned int old, unsigned int new); */ nop .if ((. - _lock_ras_start) & 15) != 12 .error "bas ras offset" .endif STATIC_LEAF_NOPROFILE(ras_atomic_cas_uint) INT_L t0, (a0) /* <- critical section start */ _atomic_cas_uint_ras_start: nop bne t0, a1, ras_atomic_cas_noupdate nop INT_S a2, (a0) /* <- critical section end */ _atomic_cas_uint_ras_end: J_RA move v0, a1 END(ras_atomic_cas_uint) /* * int _ucas_32_ras(volatile uint32_t *val, uint32_t old, uint32_t new, * uint32_t *retp); */ nop .if ((. - _lock_ras_start) & 15) != 12 .error "bas ras offset" .endif STATIC_LEAF_NOPROFILE(_ucas_32_ras) lw t0, (a0) /* <- critical section start */ _ucas_32_ras_start: nop bne t0, a1, _ucas_32_ras_end nop sw a2, (a0) /* <- critical section end */ _ucas_32_ras_end: PTR_S zero, PCB_ONFAULT(v1) J_RA sw t0, 0(a3) END(_ucas_32_ras) #ifdef _LP64 /* * int _ucas_64_ras(volatile uint64_t *val, uint64_t old, uint64_t new, * uint64_t *retp); */ .if ((. - _lock_ras_start) & 15) != 12 .error "bad ras offset" .endif STATIC_LEAF_NOPROFILE(_ucas_64_ras) ld t0, (a0) /* <- critical section start */ _ucas_64_ras_start: nop bne t0, a1, _ucas_64_ras_end nop sd a2, (a0) /* <- critical section end */ _ucas_64_ras_end: PTR_S zero, PCB_ONFAULT(v1) J_RA sd t0, 0(a3) END(_ucas_64_ras) #endif /* _LP64 */ #ifndef LOCKDEBUG /* * void ras_mutex_enter(kmutex_t *mtx); */ .if ((. - _lock_ras_start) & 15) != 12 .error "bad ras offset" .endif STATIC_LEAF_NOPROFILE(ras_mutex_enter) PTR_L t0, (a0) /* <- critical section start */ _mutex_enter_ras_start: nop bnez t0, ras_mutex_vector_enter nop PTR_S MIPS_CURLWP, (a0)/* <- critical section end */ _mutex_enter_ras_end: J_RA nop END(ras_mutex_enter) /* * int ras_mutex_exit(kmutex_t *mtx); */ nop .if ((. - _lock_ras_start) & 15) != 12 .error "bas ras offset" .endif STATIC_LEAF_NOPROFILE(ras_mutex_exit) PTR_L t0, (a0) /* <- critical section start */ _mutex_exit_ras_start: nop bne t0, MIPS_CURLWP, ras_mutex_vector_exit nop PTR_S zero, (a0) /* <- critical section end */ _mutex_exit_ras_exit: J_RA nop END(ras_mutex_exit) /* * These could moved out to fit in more RAS sequences. */ STATIC_LEAF_NOPROFILE(ras_mutex_vector_enter) j _C_LABEL(mutex_vector_enter) nop END(ras_mutex_vector_enter) STATIC_LEAF_NOPROFILE(ras_mutex_vector_exit) j _C_LABEL(mutex_vector_exit) nop END(ras_mutex_vector_exit) #endif /* !LOCKDEBUG */ .p2align LOG2_MIPS_LOCK_RAS_SIZE /* Get out of the RAS block */ .set at #ifdef MIPS3_LOONGSON2F loongson_return: j ra nop #endif /* * Patch up the given address. We arrive here if we might have trapped * within one of the critical sections above. Do: * * if ((addr & ~15) == ras) * return ras - 4; * ... check next ... * return addr; * * Registers on entry: * * k1 fault PC * ra return address * * On exit: * * k1 adjusted fault PC * ra return address * t0 clobbered * t1 clobbered */ #define RAS_MKMASK(a) (1 << (((a)-_lock_ras_start) >> 4)) /* * Since each RAS is aligned on a 16 byte boundary, we can use its offset * from _lock_ras_start to construct a bitmask of the valid RAS within. */ #ifndef LOCKDEBUG #define MUTEX_RAS_MASK (RAS_MKMASK(_mutex_enter_ras_start) \ |RAS_MKMASK(_mutex_exit_ras_start)) #else #define MUTEX_RAS_MASK 0 #endif #ifdef _LP64 #define UCAS_64_MASK RAS_MKMASK(_ucas_64_ras_start) #else #define UCAS_64_MASK 0 #endif #define RAS_MASK (RAS_MKMASK(_atomic_cas_ulong_ras_start) \ |RAS_MKMASK(_atomic_cas_uint_ras_start) \ |RAS_MKMASK(_ucas_32_ras_start) \ |UCAS_64_MASK \ |MUTEX_RAS_MASK) /* * The caller has already determined that * _lock_ras_start == (k1 & -MIPS_LOCK_RAS_SIZE) */ LEAF_NOPROFILE(_restart_lock_ras) and t0, k1, MIPS_LOCK_RAS_SIZE - 1 /* look at addr bits in ras region */ srl t0, 4 /* focus on each set of 16 bytes */ li t1, 1 /* need this to make a bitmask */ sllv t1, t1, t0 /* now we have a bitmask of the PC */ andi t1, RAS_MASK /* was the PC in a RAS? */ bnez t1, 1f /* yes, adjust PC */ and t0, k1, 15 /* get offset in RAS */ j ra nop 1: addu t0, 4 /* bias offset by one more instruction */ j ra PTR_SUBU k1, t0 /* and subtract that from the PC */ END(_restart_lock_ras) /* * int ras_ucas_32(volatile uint32_t *ptr, uint32_t old, uint32_t new, * uint32_t *retp); */ STATIC_LEAF(ras_ucas_32) PTR_L v1, L_PCB(MIPS_CURLWP) PTR_LA v0, _C_LABEL(ras_ucaserr) PTR_S v0, PCB_ONFAULT(v1) bltz a0, _C_LABEL(ras_ucaserr) nop b _C_LABEL(_ucas_32_ras) move v0, zero # assume success END(ras_ucas_32) #ifdef _LP64 /* * int ras_ucas_64(volatile uint64_t *ptr, uint64_t old, uint64_t new, * uint64_t *retp); */ STATIC_LEAF(ras_ucas_64) PTR_L v1, L_PCB(MIPS_CURLWP) PTR_LA v0, _C_LABEL(ras_ucaserr) PTR_S v0, PCB_ONFAULT(v1) bltz a0, _C_LABEL(ras_ucaserr) nop b _C_LABEL(_ucas_64_ras) move v0, zero # assume success END(ras_ucas_64) #endif /* _LP64 */ /* * */ STATIC_LEAF(ras_ucaserr) PTR_S zero, PCB_ONFAULT(v1) # reset fault handler j ra li v0, EFAULT # return EFAULT on error END(ras_ucaserr) #ifndef LOCKDEBUG /* * void mutex_spin_enter(kmutex_t *mtx); */ STATIC_NESTED(ras_mutex_spin_enter, CALLFRAME_SIZ, ra) move t0, a0 PTR_L t2, L_CPU(MIPS_CURLWP) INT_L a0, MTX_IPL(t0) #ifdef PARANOIA INT_L ta1, CPU_INFO_CPL(t2) # get current cpl #endif /* * We need to raise our IPL. * call splraise (only uses a0-a3, v0-v1, and ra) */ move t3, ra jal _C_LABEL(splraise) nop move ra, t3 /* * If this is the first lock of the mutex, store the previous IPL * for exit. */ 1: INT_L ta2, CPU_INFO_MTX_COUNT(t2) nop INT_ADDU ta3, ta2, -1 INT_S ta3, CPU_INFO_MTX_COUNT(t2) bnez ta2, 2f nop INT_S v0, CPU_INFO_MTX_OLDSPL(t2) /* returned by splraise */ 2: #if defined(DIAGNOSTIC) INT_L t3, MTX_LOCK(t0) li t1, 1 bnez t3, 3f nop j ra INT_S t1, MTX_LOCK(t0) 3: j _C_LABEL(mutex_spin_retry) move a0, t0 # a0 still holds the splraise cookie #else /* DIAGNOSTIC */ j ra nop #endif /* DIAGNOSTIC */ END(ras_mutex_spin_enter) /* * void mutex_spin_exit(kmutex_t *mtx); */ LEAF(ras_mutex_spin_exit) PTR_L t2, L_CPU(MIPS_CURLWP) nop #if defined(DIAGNOSTIC) INT_L t0, MTX_LOCK(a0) nop beqz t0, 2f nop INT_S zero, MTX_LOCK(a0) #endif /* * We need to grab this before the mutex count is incremented * because if we get an interrupt, it may see the count as zero * and overwrite the oldspl value with a bogus value. */ #ifdef PARANOIA INT_L a2, MTX_IPL(a0) #endif INT_L a0, CPU_INFO_MTX_OLDSPL(t2) /* * Increment the mutex count */ INT_L t0, CPU_INFO_MTX_COUNT(t2) nop INT_ADDU t0, t0, 1 INT_S t0, CPU_INFO_MTX_COUNT(t2) /* * If the IPL doesn't change, nothing to do */ INT_L a1, CPU_INFO_CPL(t2) nop #ifdef PARANOIA sltu v0, a1, a2 # v0 = cpl < mtx_ipl sltu v1, a1, a0 # v1 = cpl < oldspl sll v0, 1 or v0, v1 12: bnez v0, 12b # loop forever if either is true nop #endif /* PARANOIA */ beq a0, a1, 1f # if oldspl == cpl nop # no reason to drop ipl bltz t0, 1f # there are still holders nop # so don't drop IPL /* * Mutex count is zero so we need to restore the old IPL */ #ifdef PARANOIA sltiu v0, a0, IPL_HIGH+1 13: beqz v0, 13b # loop forever if ipl > IPL_HIGH nop #endif j _C_LABEL(splx) nop 1: j ra nop #if defined(DIAGNOSTIC) 2: j _C_LABEL(mutex_vector_exit) nop #endif END(ras_mutex_spin_exit) #endif /* !LOCKDEBUG */ .data EXPORT_OBJECT(mips_locore_atomicvec) PTR_WORD ras_atomic_cas_uint PTR_WORD ras_atomic_cas_ulong PTR_WORD ras_ucas_32 #ifdef _LP64 PTR_WORD ras_ucas_64 #else PTR_WORD 0 #endif /* _LP64 */ #ifdef LOCKDEBUG PTR_WORD mutex_enter PTR_WORD mutex_exit PTR_WORD mutex_spin_enter PTR_WORD mutex_spin_exit #else PTR_WORD ras_mutex_enter PTR_WORD ras_mutex_exit PTR_WORD ras_mutex_spin_enter PTR_WORD ras_mutex_spin_exit #endif /* !LOCKDEBUG */
