/*      $NetBSD: locore_mips1.S,v 1.99 2024/02/09 22:08:32 andvar Exp $ */

/*
 * Copyright (c) 1992, 1993
 *      The Regents of the University of California.  All rights reserved.
 *
 * This code is derived from software contributed to Berkeley by
 * Digital Equipment Corporation and Ralph Campbell.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 * Copyright (C) 1989 Digital Equipment Corporation.
 * Permission to use, copy, modify, and distribute this software and
 * its documentation for any purpose and without fee is hereby granted,
 * provided that the above copyright notice appears in all copies.
 * Digital Equipment Corporation makes no representations about the
 * suitability of this software for any purpose.  It is provided "as is"
 * without express or implied warranty.
 *
 * from: Header: /sprite/src/kernel/mach/ds3100.md/RCS/loMem.s,
 *      v 1.1 89/07/11 17:55:04 nelson Exp  SPRITE (DECWRL)
 * from: Header: /sprite/src/kernel/mach/ds3100.md/RCS/machAsm.s,
 *      v 9.2 90/01/29 18:00:39 shirriff Exp  SPRITE (DECWRL)
 * from: Header: /sprite/src/kernel/vm/ds3100.md/vmPmaxAsm.s,
 *      v 1.1 89/07/10 14:27:41 nelson Exp  SPRITE (DECWRL)
 *
 *      @(#)locore.s    8.5 (Berkeley) 1/4/94
 */
#include "opt_cputype.h"
#include "opt_ddb.h"
#include "opt_kgdb.h"

#include <sys/cdefs.h>

#include <mips/asm.h>
#include <mips/cpuregs.h>

RCSID("$NetBSD: locore_mips1.S,v 1.99 2024/02/09 22:08:32 andvar Exp $")

#include "assym.h"

#define _SLLV           sllv

#define _SLL            sll
#define _SRL            srl
#define WIRED_SHIFT     2

/*
 * Use correct-sized m?c0/dm?c0 opcodes.
 */
#define _MFC0   mfc0
#define _MTC0   mtc0

#if defined(__mips_n32) || defined(__mips_n64)
#error MIPS1 does not support N32/N64.
#endif

#define MIPSX(name)     __CONCAT(mips1_,name)

        .set    noreorder
        .text

EXPORT(MIPSX(exceptionentry_start))

/*
 * mipsN_utlb_miss
 *
 * A reference is made (in either kernel or user mode) to a page in
 * kuseg that has no matching TLB entry.  This routine is copied down
 * at 0x80000000 and total length must be less than 32 instructions.
 * No pc relative jump instruction is allowed.
 */
VECTOR(MIPSX(utlb_miss), unknown)
        .set    noat
        _MFC0   k0, MIPS_COP_0_BAD_VADDR        #00: k0=bad address
        lui     k1, %hi(CPUVAR(PMAP_SEG0TAB))   #01: k1=hi of seg0tab
        bltz    k0, 1f                          # R3000 chip bug
         PTR_SRL k0, SEGSHIFT-PTR_SCALESHIFT    #03: k0=seg offset (almost)
        PTR_L   k1, %lo(CPUVAR(PMAP_SEG0TAB))(k1) #04: k1=seg0tab
        andi    k0, (NSEGPG-1)<<PTR_SCALESHIFT  #07: k0=seg offset (mask 0x3)
        PTR_ADDU k1, k0                         #08: k1=seg entry address
        PTR_L   k1, 0(k1)                       #09: k1=seg entry
        _MFC0   k0, MIPS_COP_0_BAD_VADDR        #0a: k0=bad address (again)
        beqz    k1, MIPSX(nopagetable)          #0b: ==0 -- no page table
         PTR_SRL k0, (PGSHIFT-PTPSHIFT)         #0c: k0=VPN (aka va>>10)
        andi    k0, (NPTEPG-1) << PTPSHIFT      #0d: k0=page table offset
        PTR_ADDU k1, k0                         #0e: k1=pte address
        INT_L   k0, 0(k1)                       #0f: k0=lo0 pte
        nop                                     #10: load delay
        beqz    k0, MIPSX(invalidpte)           #11: dont load invalid entries
         nop                                    #12  branch delay
        mtc0    k0, MIPS_COP_0_TLB_LOW          #13: lo0 is loaded
        nop                                     #14: load delay
        tlbwr                                   #15: update TLB
1:
        _MFC0   k1, MIPS_COP_0_EXC_PC           #16: get return address
        nop                                     #17: load delay
        j       k1                              #18: return from
         rfe                                    #19:    exception
MIPSX(nopagetable):
MIPSX(invalidpte):
        j       MIPSX(slowfault)                #1a: handle the rest
         nop                                    #1b: branch delay
        .set    at
VECTOR_END(MIPSX(utlb_miss))


/*
 * mipsN_exception
 *
 * Handles any exceptions other than reset and UTLB miss.  This routine
 * is copied down at 0x80000080 and total length must be less than 32
 * instructions.  No pc relative jump instruction is allowed.
 */
        .org    MIPSX(utlb_miss) + 0x80
VECTOR(MIPSX(exception), unknown)
/*
 * Find out what mode we came from and jump to the proper handler.
 */
        .set    noat
        mfc0    k0, MIPS_COP_0_STATUS           #00: get the status register
        mfc0    k1, MIPS_COP_0_CAUSE            #01: get the cause register
        and     k0, MIPS1_SR_KU_PREV            #02: test for user mode
        sll     k0, 4                           #03: shift user bit for cause index
        and     k1, MIPS1_CR_EXC_CODE           #04: mask out the cause bits
        or      k1, k0                          #05: change index to user table
        PTR_LA  k0, MIPSX(excpt_sw)             #06: get base of the jump table
        PTR_ADDU k0, k1                         #08: get the address of the
                                                #  function entry.  Note that
                                                #  the cause is already
                                                #  shifted left by 2 bits so
                                                #  we dont have to shift.
        PTR_L   k0, 0(k0)                       #09: get the function address
        nop                                     #0a: load delay
        j       k0                              #0b: jump to the function
         nop                                    #0c
        nop                                     #0d
        nop                                     #0e
        nop                                     #0f
        .set    at
VECTOR_END(MIPSX(exception))


/*----------------------------------------------------------------------------
 *
 * mipsN_slowfault
 *
 * Alternate entry point into the mipsN_user_gen_exception or
 * mipsN_kern_gen_exception, when the UTLB miss handler couldn't
 * find a TLB entry.
 *
 * Find out what mode we came from and call the appropriate handler.
 *
 *----------------------------------------------------------------------------
 */
MIPSX(slowfault):
        .set    noat
        mfc0    k0, MIPS_COP_0_STATUS
        nop
        and     k0, MIPS1_SR_KU_PREV
        bnez    k0, _C_LABEL(MIPSX(user_gen_exception))
         nop
        .set    at
/*
 * Fall through ...
 */

/*
 * mipsN_kern_gen_exception
 *
 * Handle an exception during kernel mode.
 * Build trapframe on stack to hold interrupted kernel context, then
 * call trap() to process the condition.
 *
 * trapframe is pointed to by the 5th arg and a dummy sixth argument is used
 * to avoid alignment problems
 * {
 *      register_t cf_args[4 + 1];
 *      register_t cf_pad;              (for 8 word alignment)
 *      register_t cf_sp;
 *      register_t cf_ra;
 *      struct reg cf_tf;
 * };
 */
NESTED_NOPROFILE(MIPSX(kern_gen_exception), KERNFRAME_SIZ, ra)
        .set    noat
        .mask   0x80000000, -4
#ifdef PARANOIA
        PTR_L   k0, L_PCB(MIPS_CURLWP)
        nop
        slt     k0, k0, sp              # k0 = L_PCB(MIPS_CURLWP) < sp
1:      beqz    k0, 1b                  # loop forever if false
         nop
        PTR_L   k0, L_PCB(MIPS_CURLWP)
        nop
        PTR_ADDU k0, USPACE
        slt     k0, sp, k0              # k0 = sp < L_PCB(MIPS_CURLWP) + USPACE
2:      beqz    k0, 2b                  # loop forever if false
         nop
#endif /* PARANOIA
/*
 * Save the relevant kernel registers onto the stack.
 * We don't need to save s0 - s8, sp and gp because
 * the compiler does it for us.
 */
        PTR_SUBU sp, KERNFRAME_SIZ
        REG_S   AT, TF_BASE+TF_REG_AST(sp)
        REG_S   v0, TF_BASE+TF_REG_V0(sp)
        REG_S   v1, TF_BASE+TF_REG_V1(sp)
        mflo    v0
        mfhi    v1
        REG_S   a0, TF_BASE+TF_REG_A0(sp)
        REG_S   a1, TF_BASE+TF_REG_A1(sp)
        REG_S   a2, TF_BASE+TF_REG_A2(sp)
        REG_S   a3, TF_BASE+TF_REG_A3(sp)
        mfc0    a0, MIPS_COP_0_STATUS           # 1st arg is STATUS
        REG_S   t0, TF_BASE+TF_REG_T0(sp)
        REG_S   t1, TF_BASE+TF_REG_T1(sp)
        REG_S   t2, TF_BASE+TF_REG_T2(sp)
        REG_S   t3, TF_BASE+TF_REG_T3(sp)
        mfc0    a1, MIPS_COP_0_CAUSE            # 2nd arg is CAUSE
        REG_S   ta0, TF_BASE+TF_REG_TA0(sp)
        REG_S   ta1, TF_BASE+TF_REG_TA1(sp)
        REG_S   ta2, TF_BASE+TF_REG_TA2(sp)
        REG_S   ta3, TF_BASE+TF_REG_TA3(sp)
        _MFC0   a2, MIPS_COP_0_BAD_VADDR        # 3rd arg is fault address
        #REG_S  t8, TF_BASE+TF_REG_T8(sp)       # is MIPS_CURLWP
        REG_S   t9, TF_BASE+TF_REG_T9(sp)
        REG_S   ra, TF_BASE+TF_REG_RA(sp)
        REG_S   a0, TF_BASE+TF_REG_SR(sp)
        _MFC0   a3, MIPS_COP_0_EXC_PC           # 4th arg is exception PC
        REG_S   v0, TF_BASE+TF_REG_MULLO(sp)
        REG_S   v1, TF_BASE+TF_REG_MULHI(sp)
        REG_S   a3, TF_BASE+TF_REG_EPC(sp)
        REG_S   a1, TF_BASE+TF_REG_CAUSE(sp)
#if defined(DDB) || defined(KGDB)
        REG_S   s0, TF_BASE+TF_REG_S0(sp)
        REG_S   s1, TF_BASE+TF_REG_S1(sp)
        REG_S   s2, TF_BASE+TF_REG_S2(sp)
        REG_S   s3, TF_BASE+TF_REG_S3(sp)
        REG_S   s4, TF_BASE+TF_REG_S4(sp)
        REG_S   s5, TF_BASE+TF_REG_S5(sp)
        REG_S   s6, TF_BASE+TF_REG_S6(sp)
        REG_S   s7, TF_BASE+TF_REG_S7(sp)
        PTR_ADDU v0, sp, KERNFRAME_SIZ
        REG_S   v0, TF_BASE+TF_REG_SP(sp)
        REG_S   s8, TF_BASE+TF_REG_S8(sp)
        REG_S   gp, TF_BASE+TF_REG_GP(sp)
#endif
        PTR_ADDU v0, sp, TF_BASE
        REG_S   v0, KERNFRAME_ARG5(sp)          # 5th arg is p. to trapframe
#ifdef PARANOIA
        /*
         * save PPL in trapframe
         */
        PTR_L   t0, L_CPU(MIPS_CURLWP)
        nop
        INT_L   t1, CPU_INFO_CPL(t0)            # get current priority level
        nop
        INT_S   t1, TF_BASE+TF_PPL(sp)          # save priority level
#endif /* PARANOIA */

#if defined(DDB) || defined(DEBUG) || defined(KGDB)
        PTR_ADDU v0, sp, KERNFRAME_SIZ
        REG_S   v0, KERNFRAME_SP(sp)
#endif

#ifdef PARANOIA
        /*
         * Verify our existing interrupt level.
         */
        jal     _C_LABEL(splcheck)
         nop
#endif /* PARANOIA */

        /*
         * Call the trap handler.
         */
        jal     _C_LABEL(trap)
         REG_S  a3, KERNFRAME_RA(sp)            # for debugging

        /*
         * Restore registers and return from the exception.
         */
        REG_L   a0, TF_BASE+TF_REG_SR(sp)
        nop
        mtc0    a0, MIPS_COP_0_STATUS           # restore the SR, disable intrs

        /*
         * Start of common kernel exception return code for both
         * mipxN_kern_gen_exception and mipsN_kern_intr.
         */
MIPSX(kern_return):
        REG_L   t0, TF_BASE+TF_REG_MULLO(sp)
        REG_L   t1, TF_BASE+TF_REG_MULHI(sp)
        REG_L   k1, TF_BASE+TF_REG_EPC(sp)      # might be changed inside trap
        mtlo    t0
        mthi    t1

#ifdef PARANOIA
        INT_L   t2, TF_BASE+TF_PPL(sp)          # get saved priority level
        PTR_L   t0, L_CPU(MIPS_CURLWP)
        nop
        INT_L   t1, CPU_INFO_CPL(t0)            # get current priority level
        nop
11:     bne     t2, t1, 11b                     # loop forever if unequal
         nop

        /*
         * Verify our existing interrupt level.
         */
        jal     _C_LABEL(splcheck)
         nop
#endif /* PARANOIA */

        /*
         * Check for kernel restartable atomic sequences.
         */
        PTR_LA  t0, _C_LABEL(_lock_ras_start)
        li      t1, -MIPS_LOCK_RAS_SIZE
        and     t1, k1
        bne     t1, t0, 1f                      # exception PC in RAS area?
         nop
        jal     _C_LABEL(_restart_lock_ras)     # fix the pc (k1)
         nop
1:

        REG_L   AT, TF_BASE+TF_REG_AST(sp)
        REG_L   v0, TF_BASE+TF_REG_V0(sp)
        REG_L   v1, TF_BASE+TF_REG_V1(sp)
        REG_L   a0, TF_BASE+TF_REG_A0(sp)
        REG_L   a1, TF_BASE+TF_REG_A1(sp)
        REG_L   a2, TF_BASE+TF_REG_A2(sp)
        REG_L   a3, TF_BASE+TF_REG_A3(sp)
        REG_L   t0, TF_BASE+TF_REG_T0(sp)
        REG_L   t1, TF_BASE+TF_REG_T1(sp)
        REG_L   t2, TF_BASE+TF_REG_T2(sp)
        REG_L   t3, TF_BASE+TF_REG_T3(sp)
        REG_L   ta0, TF_BASE+TF_REG_TA0(sp)
        REG_L   ta1, TF_BASE+TF_REG_TA1(sp)
        REG_L   ta2, TF_BASE+TF_REG_TA2(sp)
        REG_L   ta3, TF_BASE+TF_REG_TA3(sp)
        #REG_L  t8, TF_BASE+TF_REG_T8(sp)       # is MIPS_CURLWP
        REG_L   t9, TF_BASE+TF_REG_T9(sp)
        REG_L   ra, TF_BASE+TF_REG_RA(sp)
#ifdef DDBnotyet
        REG_L   s0, TF_BASE+TF_REG_S0(sp)
        REG_L   s1, TF_BASE+TF_REG_S1(sp)
        REG_L   s2, TF_BASE+TF_REG_S2(sp)
        REG_L   s3, TF_BASE+TF_REG_S3(sp)
        REG_L   s4, TF_BASE+TF_REG_S4(sp)
        REG_L   s5, TF_BASE+TF_REG_S5(sp)
        REG_L   s6, TF_BASE+TF_REG_S6(sp)
        REG_L   s7, TF_BASE+TF_REG_S7(sp)
        REG_L   s8, TF_BASE+TF_REG_S8(sp)
#endif
        PTR_ADDU sp, KERNFRAME_SIZ
        j       k1                              # return to interrupted point
        rfe
        .set    at
END(MIPSX(kern_gen_exception))

/*
 * mipsN_kern_intr
 *
 * Handle an interrupt from kernel mode.
 * Build kernframe on stack to hold interrupted kernel context, then
 * call cpu_intr() to process it.
 *
 */
NESTED_NOPROFILE(MIPSX(kern_intr), KERNFRAME_SIZ, ra)
        .set    noat
        .mask   0x80000000, -4
#ifdef PARANOIA
        PTR_L   k0, L_PCB(MIPS_CURLWP)
        nop
        slt     k0, k0, sp                      # k0 = L_PCB(MIPS_CURLWP) < sp
1:      beqz    k0, 1b                          # loop forever if false
         nop
        PTR_L   k0, L_PCB(MIPS_CURLWP)
        nop
        PTR_ADDU k0, USPACE
        slt     k0, sp, k0                      # k0 = sp < L_PCB(MIPS_CURLWP) + USPACE
2:      beqz    k0, 2b                          # loop forever if false
         nop
        PTR_L   k0, L_CPU(MIPS_CURLWP)
        nop
        INT_L   k0, CPU_INFO_IDEPTH(k0)         # grab interrupt depth
        nop
        sltu    k0, k0, 3                       # must be < 3
3:      beqz    k0, 3b                          # loop forever if false
         nop
#endif
        /*
         * Save the relevant kernel registers onto the stack.  We don't need
         * to save s0 - s8, sp, and gp because the compiler does it for us.
         * But we use s0-s2 so need to save them.
         */
        PTR_SUBU sp, KERNFRAME_SIZ
        REG_S   AT, TF_BASE+TF_REG_AST(sp)
        REG_S   v0, TF_BASE+TF_REG_V0(sp)
        REG_S   v1, TF_BASE+TF_REG_V1(sp)
        mflo    v0
        mfhi    v1
        REG_S   a0, TF_BASE+TF_REG_A0(sp)
        REG_S   a1, TF_BASE+TF_REG_A1(sp)
        REG_S   a2, TF_BASE+TF_REG_A2(sp)
        REG_S   a3, TF_BASE+TF_REG_A3(sp)
        REG_S   t0, TF_BASE+TF_REG_T0(sp)
        REG_S   t1, TF_BASE+TF_REG_T1(sp)
        REG_S   t2, TF_BASE+TF_REG_T2(sp)
        REG_S   t3, TF_BASE+TF_REG_T3(sp)
        REG_S   ta0, TF_BASE+TF_REG_TA0(sp)
        REG_S   ta1, TF_BASE+TF_REG_TA1(sp)
        REG_S   ta2, TF_BASE+TF_REG_TA2(sp)
        REG_S   ta3, TF_BASE+TF_REG_TA3(sp)
        REG_S   s0, TF_BASE+TF_REG_S0(sp)       # used for saved ipl/idepth
        REG_S   s1, TF_BASE+TF_REG_S1(sp)       # used for initial status
        mfc0    s1, MIPS_COP_0_STATUS
        REG_S   s2, TF_BASE+TF_REG_S2(sp)       # used for cpu_info
        #REG_S  t8, TF_BASE+TF_REG_T8(sp)       # already contains MIPS_CURLWP
        REG_S   t9, TF_BASE+TF_REG_T9(sp)
        REG_S   ra, TF_BASE+TF_REG_RA(sp)
        REG_S   s1, TF_BASE+TF_REG_SR(sp)
        REG_S   v0, TF_BASE+TF_REG_MULLO(sp)
        REG_S   v1, TF_BASE+TF_REG_MULHI(sp)
/*
 * Call the interrupt handler.
 */
        _MFC0   ta0, MIPS_COP_0_EXC_PC          # grab exception PC
        PTR_L   s2, L_CPU(MIPS_CURLWP)          # delay slot
        REG_S   ta0, TF_BASE+TF_REG_EPC(sp)     # and save it

#if defined(DDB) || defined(DEBUG) || defined(KGDB)
        REG_S   ta0, KERNFRAME_RA(sp)           # for debugging
#endif

#ifdef PARANOIA
        INT_L   s0, CPU_INFO_CPL(s2)
        nop                                     # load delay
        INT_S   s0, TF_BASE+TF_PPL(sp)          # save priority level

        /*
         * Verify the current interrupt level
         */
        jal     _C_LABEL(splcheck)
         nop
#endif /* PARANOIA */

        /*
         * We first need to get to IPL_HIGH so that interrupts are masked.
         */
        jal     _C_LABEL(splhigh_noprof)
         nop

#ifdef PARANOIA
1:      bne     s0, v0, 1b
         nop
#endif /* PARANOIA */

        sll     s0, v0, 8                       # remember previous priority
                                                # low 8 bits used for idepth

#ifdef PARANOIA
        /*
         * Interrupts at IPL_HIGH are not allowed.
         */
        li      v1, IPL_HIGH
        sltu    t0, v0, v1
2:      beqz    t0, 2b
         nop
#endif /* PARANOIA */

        INT_L   t1, CPU_INFO_IDEPTH(s2)         # we need to inc. intr depth
        nop                                     # load delay
        or      s0, t1                          #   save old interrupt depth
        INT_ADDU t1, 1
        INT_S   t1, CPU_INFO_IDEPTH(s2)         #   store new interrupt depth

        /*
         * Now that we're at splhigh so all interrupts are masked
         * individually and we won't get interrupted here, turn the
         * global interrupt enable bit on again. This will allow
         * high-priority interrupts to be delivered once a
         * low-priority interrupt handler lowers spl to execute.
         */
        mfc0    v1, MIPS_COP_0_STATUS
        nop
        or      v0, v1, MIPS_SR_INT_IE
        mtc0    v0, MIPS_COP_0_STATUS           # write new status

        /*
         * Now hard interrupts can be processed.
         */
        move    a1, ta0                         # 2nd arg is exception PC
        move    a2, s1                          # 3rd arg is status
        jal     _C_LABEL(cpu_intr)              # cpu_intr(ppl, pc, status)
         srl    a0, s0, 8                       # 1st arg is previous pri level

        and     t1, s0, 0xff                    # get previous interrupt depth
        INT_S   t1, CPU_INFO_IDEPTH(s2)         # to it previous value

#ifdef PARANOIA
        mfc0    t0, MIPS_COP_0_STATUS           # verify INT_IE is still set
        nop
        and     t0, MIPS_SR_INT_IE
3:      beqz    t0, 3b
         nop
#endif /* PARANOIA */

#ifdef __HAVE_FAST_SOFTINTS
        and     a0, s1, MIPS_SOFT_INT_MASK      # were softints enabled?
        beqz    a0, 4f                          #   nope
         nop
        mfc0    v0, MIPS_COP_0_CAUSE            # grab the pending softints
        nop
        and     a0, v0                          # are softints pending
        beqz    a0, 4f                          #   nope
         nop

        jal     _C_LABEL(softint_process)       # softint_process(pending)
         nop

#ifdef __HAVE_PREEMPTION
        srl     v1, s0, 8                       # get saved priority level
        bnez    v1, 4f                          # branch if not at IPL_NONE
         nop
        INT_L   t0, CPU_INFO_SOFTINTS(s2)       # get pending softints
        nop
        and     v0, t0, 1 << SOFTINT_KPREEMPT   # do we need a kernel preempt?
        beqz    v0, 4f                          #   nope
         nop
        xor     t0, v0                          # clear preempt bit
        INT_S   t0, CPU_INFO_SOFTINTS(s2)       # and save it.
        jal     _C_LABEL(kpreempt)              # kpreempt(pc)
         PTR_L  a0, TF_BASE+TF_REG_EPC(sp)
#endif /* __HAVE_PREEMPTION */
4:
#endif /* __HAVE_FAST_SOFTINTS */
        /*
         * Interrupts handled, restore registers and return from the interrupt.
         * First, clear interrupt enable
         */
        mtc0    s1, MIPS_COP_0_STATUS           # disable interrupts

        srl     a0, s0, 8                       # get previous priority level
#ifdef PARANOIA
        INT_L   t0, TF_BASE+TF_PPL(sp)          # get saved priority level
        nop
9:      bne     t0, a0, 9b                      # should still match
         nop

        li      t0, IPL_HIGH
        sltu    v0, a0, t0
8:      beqz    v0, 8b
         nop
#endif /* PARANOIA */

        /*
         * Restore IPL knowing interrupts are disabled
         */
        jal     _C_LABEL(splx_noprof)           # splx(ppl)
         nop

#ifdef PARANOIA
        mfc0    v0, MIPS_COP_0_STATUS
        nop
        or      v0, MIPS_SR_INT_IE
5:      bne     v0, s1, 5b
         nop
#endif /* PARANOIA */

        /*
         * Restore SR
         */
        mtc0    s1, MIPS_COP_0_STATUS

        /*
         * Restore s0-s2 and goto common kernel return code.
         */
        REG_L   s0, TF_BASE+TF_REG_S0(sp)
        REG_L   s1, TF_BASE+TF_REG_S1(sp)
        b       MIPSX(kern_return)
         REG_L  s2, TF_BASE+TF_REG_S2(sp)
        .set    at
END(MIPSX(kern_intr))

/*
 * mipsN_user_gen_exception
 *
 * Handle an exception during user mode.
 * Save user context atop the kernel stack, then call trap() to process
 * the condition.  The context can be manipulated alternatively via
 * curlwp->l_md.md_regs.
 */
NESTED_NOPROFILE(MIPSX(user_gen_exception), CALLFRAME_SIZ, ra)
        .set    noat
        .mask   0x80000000, -4
        /*
         * Save all the registers except the kernel temporaries onto the stack.
         */
        PTR_L   k1, CPUVAR(CURLWP)
        nop
        PTR_L   k0, L_PCB(k1)
        nop
        PTR_ADDU k0, USPACE - TF_SIZ - CALLFRAME_SIZ
        REG_S   AT, CALLFRAME_SIZ+TF_REG_AST(k0)
        REG_S   v0, CALLFRAME_SIZ+TF_REG_V0(k0)
        REG_S   v1, CALLFRAME_SIZ+TF_REG_V1(k0)
        mflo    v0
        REG_S   a0, CALLFRAME_SIZ+TF_REG_A0(k0)
        REG_S   a1, CALLFRAME_SIZ+TF_REG_A1(k0)
        REG_S   a2, CALLFRAME_SIZ+TF_REG_A2(k0)
        REG_S   a3, CALLFRAME_SIZ+TF_REG_A3(k0)
        mfhi    v1
        REG_S   t0, CALLFRAME_SIZ+TF_REG_T0(k0)
        REG_S   t1, CALLFRAME_SIZ+TF_REG_T1(k0)
        REG_S   t2, CALLFRAME_SIZ+TF_REG_T2(k0)
        REG_S   t3, CALLFRAME_SIZ+TF_REG_T3(k0)
        mfc0    a0, MIPS_COP_0_STATUS           # 1st arg is STATUS
        REG_S   ta0, CALLFRAME_SIZ+TF_REG_TA0(k0)
        REG_S   ta1, CALLFRAME_SIZ+TF_REG_TA1(k0)
        REG_S   ta2, CALLFRAME_SIZ+TF_REG_TA2(k0)
        REG_S   ta3, CALLFRAME_SIZ+TF_REG_TA3(k0)
        mfc0    a1, MIPS_COP_0_CAUSE            # 2nd arg is CAUSE
        REG_S   s0, CALLFRAME_SIZ+TF_REG_S0(k0)
        REG_S   s1, CALLFRAME_SIZ+TF_REG_S1(k0)
        REG_S   s2, CALLFRAME_SIZ+TF_REG_S2(k0)
        REG_S   s3, CALLFRAME_SIZ+TF_REG_S3(k0)
        _MFC0   a2, MIPS_COP_0_BAD_VADDR        # 3rd arg is fault address
        REG_S   s4, CALLFRAME_SIZ+TF_REG_S4(k0)
        REG_S   s5, CALLFRAME_SIZ+TF_REG_S5(k0)
        REG_S   s6, CALLFRAME_SIZ+TF_REG_S6(k0)
        REG_S   s7, CALLFRAME_SIZ+TF_REG_S7(k0)
        _MFC0   a3, MIPS_COP_0_EXC_PC           # 4th arg is exception PC
        REG_S   t8, CALLFRAME_SIZ+TF_REG_T8(k0) # will be MIPS_CURLWP
        REG_S   t9, CALLFRAME_SIZ+TF_REG_T9(k0)
        REG_S   gp, CALLFRAME_SIZ+TF_REG_GP(k0)
        REG_S   sp, CALLFRAME_SIZ+TF_REG_SP(k0)
        REG_S   s8, CALLFRAME_SIZ+TF_REG_S8(k0)
        REG_S   ra, CALLFRAME_SIZ+TF_REG_RA(k0)
        REG_S   a0, CALLFRAME_SIZ+TF_REG_SR(k0)
        REG_S   v0, CALLFRAME_SIZ+TF_REG_MULLO(k0)
        REG_S   v1, CALLFRAME_SIZ+TF_REG_MULHI(k0)
        REG_S   a3, CALLFRAME_SIZ+TF_REG_EPC(k0)
#ifdef __GP_SUPPORT__
        PTR_LA  gp, _C_LABEL(_gp)               # switch to kernel GP
#endif
        move    sp, k0                          # switch to kernel SP
        move    MIPS_CURLWP, k1
#ifndef NOFPU
        lui     t0, %hi(MIPS_SR_COP_1_BIT)
        and     t0, a0
        beqz    t0, 1f
         xor    t0, a0                          # turn off the FPU
        mtc0    t0, MIPS_COP_0_STATUS
         nop
1:
#endif
/*
 * Call the trap handler.
 */
        jal     _C_LABEL(trap)
         REG_S  a3, CALLFRAME_RA(sp)            # for debugging
/*
 * Check pending asynchronous traps.
 */
        INT_L   v0, L_MD_ASTPENDING(MIPS_CURLWP)# any pending ast?
        nop
        beqz    v0, MIPSX(user_return)          # if no, skip ast processing
         nop
/*
 * We have pending asynchronous traps; all the state is already saved.
 */
        lui     ra, %hi(MIPSX(user_return))     # return directly to user return
        j       _C_LABEL(ast)
         PTR_ADDIU ra, %lo(MIPSX(user_return))  # return directly to user return
        .set    at
END(MIPSX(user_gen_exception))

/*----------------------------------------------------------------------------
 *
 * mipsN_user_intr
 *
 *      Handle an interrupt from user mode.
 *      We save partial state onto the kernel stack since we know there will
 *      always a kernel stack and chances are we won't need the registers we
 *      don't save.  If there is a pending asynchronous system trap, then save
 *      the remaining state and call ast().
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------------
 */
NESTED_NOPROFILE(MIPSX(user_intr), CALLFRAME_SIZ, ra)
        .set    noat
        .mask   0x80000000, -4
/*
 * Save the relevant user registers onto the kernel stack.
 * We don't need to save s0 - s8 because the compiler does it for us.
 */
        PTR_L   k1, CPUVAR(CURLWP)
        nop
        PTR_L   k0, L_PCB(k1)                           # XXXuvm_lwp_getuarea
        nop
        PTR_ADDU k0, USPACE - TF_SIZ - CALLFRAME_SIZ
        REG_S   AT, CALLFRAME_SIZ+TF_REG_AST(k0)        # $1
        REG_S   v0, CALLFRAME_SIZ+TF_REG_V0(k0)         # $2
        REG_S   v1, CALLFRAME_SIZ+TF_REG_V1(k0)         # $3
        mflo    v0
        REG_S   a0, CALLFRAME_SIZ+TF_REG_A0(k0)         # $4
        REG_S   a1, CALLFRAME_SIZ+TF_REG_A1(k0)         # $5
        REG_S   a2, CALLFRAME_SIZ+TF_REG_A2(k0)         # $6
        REG_S   a3, CALLFRAME_SIZ+TF_REG_A3(k0)         # $7
        mfhi    v1
        REG_S   t0, CALLFRAME_SIZ+TF_REG_T0(k0)         # $8
        REG_S   t1, CALLFRAME_SIZ+TF_REG_T1(k0)         # $9
        REG_S   t2, CALLFRAME_SIZ+TF_REG_T2(k0)         # $10
        REG_S   t3, CALLFRAME_SIZ+TF_REG_T3(k0)         # $11
        mfc0    t0, MIPS_COP_0_CAUSE
        REG_S   ta0, CALLFRAME_SIZ+TF_REG_TA0(k0)       # $12
        REG_S   ta1, CALLFRAME_SIZ+TF_REG_TA1(k0)       # $13
        REG_S   ta2, CALLFRAME_SIZ+TF_REG_TA2(k0)       # $14
        REG_S   ta3, CALLFRAME_SIZ+TF_REG_TA3(k0)       # $15
        REG_S   s0, CALLFRAME_SIZ+TF_REG_S0(k0)         # $16
        REG_S   s1, CALLFRAME_SIZ+TF_REG_S1(k0)         # $17
        mfc0    s1, MIPS_COP_0_STATUS
        REG_S   t8, CALLFRAME_SIZ+TF_REG_T8(k0)         # $24 MIPS_CURLWP
        REG_S   t9, CALLFRAME_SIZ+TF_REG_T9(k0)         # $25
        REG_S   gp, CALLFRAME_SIZ+TF_REG_GP(k0)         # $28
        REG_S   sp, CALLFRAME_SIZ+TF_REG_SP(k0)         # $29
        REG_S   ra, CALLFRAME_SIZ+TF_REG_RA(k0)         # $31
        REG_S   s1, CALLFRAME_SIZ+TF_REG_SR(k0)
        _MFC0   ta0, MIPS_COP_0_EXC_PC
        REG_S   v0, CALLFRAME_SIZ+TF_REG_MULLO(k0)
        REG_S   v1, CALLFRAME_SIZ+TF_REG_MULHI(k0)
        REG_S   ta0, CALLFRAME_SIZ+TF_REG_EPC(k0)
        REG_S   t0, CALLFRAME_SIZ+TF_REG_CAUSE(k0)
        move    sp, k0                          # switch to kernel SP
        move    MIPS_CURLWP, k1                 # set curlwp reg (t8)
#if defined(DDB) || defined(DEBUG) || defined(KGDB)
        REG_S   ta0, CALLFRAME_RA(sp)           # for debugging
#endif
#ifdef __GP_SUPPORT__
        PTR_LA  gp, _C_LABEL(_gp)               # switch to kernel GP
#endif

        /*
         * We first need to get to IPL_HIGH so that interrupts are masked.
         */
        jal     _C_LABEL(splhigh_noprof)        # splhigh()
         nop
        move    s0, v0                          # remember previous priority

        /*
         * Now that we're at splhigh so all interrupts are masked
         * individually and we won't get interrupted here, turn the
         * global interrupt enable bit on again. This will allow
         * high-priority interrupts to be delivered once a
         * low-priority interrupt handler lowers spl to execute.
         *
         * Also switch off the FPU.
         */
        mfc0    v1, MIPS_COP_0_STATUS
#ifndef NOFPU
        lui     v0, %hi(MIPS_SR_COP_1_BIT)
        and     v0, v1
        or      v0, MIPS_SR_INT_IE              # make sure intrs are still on
#else
        li      v0, MIPS_SR_INT_IE              # reenable intrs
#endif
        xor     v0, v1
        mtc0    v0, MIPS_COP_0_STATUS
        nop

        /*
         * Since we interrupted user mode, the new interrupt depth must be 1.
         */
        PTR_L   t0, L_CPU(MIPS_CURLWP)
        li      t1, 1
        INT_S   t1, CPU_INFO_IDEPTH(t0)         # store new interrupt depth (1)

        /*
         * Now hard interrupts can be processed.
         */
        move    a1, ta0                         # 2nd arg is exception pc
        move    a2, s1                          # 3rd arg is status
        jal     _C_LABEL(cpu_intr)              # cpu_intr(ppl, pc, status)
         move   a0, s0                          # 1st arg is previous pri level

        /*
         * Interrupt depth is now back to 0.
         */
        PTR_L   t0, L_CPU(MIPS_CURLWP)
        nop
        INT_S   zero, CPU_INFO_IDEPTH(t0)

#ifdef __HAVE_FAST_SOFTINTS
        /*
         * This an interrupt from user mode so both softints must be enabled.
         * No need to check (unless we're being paranoid).
         */
#ifdef PARANOIA
        and     a0, s1, MIPS_SOFT_INT_MASK      # get softints enabled bits
        xor     a0, MIPS_SOFT_INT_MASK          # invert them.
1:      bnez    a0, 1b                          # loop forever if disabled
         nop
#endif
        mfc0    a0, MIPS_COP_0_CAUSE            # grab the pending softints
        nop                                     # load delay
        and     a0, MIPS_SOFT_INT_MASK          # are there softints pending
        beqz    a0, 4f                          #   nope
         nop
        jal     _C_LABEL(softint_process)       # softint_process(pending)
         nop
4:
#endif
        /*
         * Disable interrupts
         */
        mfc0    v1, MIPS_COP_0_STATUS
        nop                                     # delay slot
        and     v0, v1, MIPS_SR_INT_IE          # clear interrupt enable
        xor     v0, v1
        mtc0    v0, MIPS_COP_0_STATUS           # interrupts are disabled

        /*
         * Restore IPL knowing interrupts are off
         */
        jal     _C_LABEL(splx_noprof)
         move   a0, s0                          # fetch previous priority level

        /*
         * Check pending asynchronous traps.
         */
        REG_L   s0, CALLFRAME_SIZ+TF_REG_S0(sp) # restore
        REG_L   s1, CALLFRAME_SIZ+TF_REG_S1(sp) # restore
        INT_L   v0, L_MD_ASTPENDING(MIPS_CURLWP)# any pending ast?
        nop
        beqz    v0, MIPSX(user_intr_return)     # if no, skip ast processing
         nop

        /*
         * We have a pending asynchronous trap; save remaining user state into
         * trapframe.
         */
        #REG_S  s0, CALLFRAME_SIZ+TF_REG_S0(sp) # $16 (saved above)
        #REG_S  s1, CALLFRAME_SIZ+TF_REG_S1(sp) # $17 (saved above)
        REG_S   s2, CALLFRAME_SIZ+TF_REG_S2(sp) # $18
        REG_S   s3, CALLFRAME_SIZ+TF_REG_S3(sp) # $19
        REG_S   s4, CALLFRAME_SIZ+TF_REG_S4(sp) # $20
        REG_S   s5, CALLFRAME_SIZ+TF_REG_S5(sp) # $21
        REG_S   s6, CALLFRAME_SIZ+TF_REG_S6(sp) # $22
        REG_S   s7, CALLFRAME_SIZ+TF_REG_S7(sp) # $23
        REG_S   s8, CALLFRAME_SIZ+TF_REG_S8(sp) # $30

        mfc0    t0, MIPS_COP_0_STATUS
        PTR_LA  ra, MIPSX(user_return)          # load delay
        or      t0, MIPS_SR_INT_IE              # enable interrupts
        j       _C_LABEL(ast)                   # ast()
         mtc0   t0, MIPS_COP_0_STATUS           # enable interrupts (spl0)
        .set    at
END(MIPSX(user_intr))

/*
 * mipsN_systemcall
 *
 * Save user context atop of kernel stack, then call syscall() to process
 * a system call.  The context can be manipulated alternatively via
 * curlwp->l_md.md_utf->tf_regs.
 */
NESTED_NOPROFILE(MIPSX(systemcall), CALLFRAME_SIZ, ra)
        .set    noat
        .mask   0x80000000, -4
        /*
         * Save all the registers but kernel temporaries onto the stack.
         */
        PTR_L   k1, CPUVAR(CURLWP)
        nop
        PTR_L   k0, L_PCB(k1)
        nop
        PTR_ADDU k0, USPACE - TF_SIZ - CALLFRAME_SIZ
        #REG_S  AT, CALLFRAME_SIZ+TF_REG_AST(k0)
        #.set   at
        REG_S   v0, CALLFRAME_SIZ+TF_REG_V0(k0)         # syscall #
        REG_S   v1, CALLFRAME_SIZ+TF_REG_V1(k0)         # used by syscall()
        mflo    v0
        REG_S   a0, CALLFRAME_SIZ+TF_REG_A0(k0)
        REG_S   a1, CALLFRAME_SIZ+TF_REG_A1(k0)
        REG_S   a2, CALLFRAME_SIZ+TF_REG_A2(k0)
        REG_S   a3, CALLFRAME_SIZ+TF_REG_A3(k0)
        move    a0, k1                                  # 1st arg is curlwp
        mfhi    v1
        mfc0    a1, MIPS_COP_0_STATUS                   # 2nd arg is STATUS
        REG_S   s0, CALLFRAME_SIZ+TF_REG_S0(k0)
        REG_S   s1, CALLFRAME_SIZ+TF_REG_S1(k0)
        REG_S   s2, CALLFRAME_SIZ+TF_REG_S2(k0)
        REG_S   s3, CALLFRAME_SIZ+TF_REG_S3(k0)
        mfc0    a2, MIPS_COP_0_CAUSE                    # 3rd arg is CAUSE
        REG_S   s4, CALLFRAME_SIZ+TF_REG_S4(k0)
        REG_S   s5, CALLFRAME_SIZ+TF_REG_S5(k0)
        REG_S   s6, CALLFRAME_SIZ+TF_REG_S6(k0)
        REG_S   s7, CALLFRAME_SIZ+TF_REG_S7(k0)
        _MFC0   a3, MIPS_COP_0_EXC_PC                   # 4th arg is PC
        REG_S   t0, CALLFRAME_SIZ+TF_REG_T0(k0)
        REG_S   t1, CALLFRAME_SIZ+TF_REG_T1(k0)
        REG_S   t2, CALLFRAME_SIZ+TF_REG_T2(k0)
        REG_S   t3, CALLFRAME_SIZ+TF_REG_T3(k0)         # syscall saved gp for fork
        REG_S   ta0, CALLFRAME_SIZ+TF_REG_TA0(k0)
        REG_S   ta1, CALLFRAME_SIZ+TF_REG_TA1(k0)
        REG_S   ta2, CALLFRAME_SIZ+TF_REG_TA2(k0)
        REG_S   ta3, CALLFRAME_SIZ+TF_REG_TA3(k0)
        REG_S   t8, CALLFRAME_SIZ+TF_REG_T8(k0)         # will be MIPS_CURLWP
        REG_S   t9, CALLFRAME_SIZ+TF_REG_T9(k0)
        REG_S   gp, CALLFRAME_SIZ+TF_REG_GP(k0)
        REG_S   sp, CALLFRAME_SIZ+TF_REG_SP(k0)
        REG_S   s8, CALLFRAME_SIZ+TF_REG_S8(k0)
        REG_S   ra, CALLFRAME_SIZ+TF_REG_RA(k0)
        REG_S   a1, CALLFRAME_SIZ+TF_REG_SR(k0)
        REG_S   v0, CALLFRAME_SIZ+TF_REG_MULLO(k0)
        REG_S   v1, CALLFRAME_SIZ+TF_REG_MULHI(k0)
        REG_S   a3, CALLFRAME_SIZ+TF_REG_EPC(k0)
        PTR_L   t0, L_PROC(a0)                  # curlwp->l_proc (used below)
        move    sp, k0                          # switch to kernel SP
        move    MIPS_CURLWP, a0                 # set curlwp reg
#ifdef __GP_SUPPORT__
        PTR_LA  gp, _C_LABEL(_gp)               # switch to kernel GP
#endif
#if defined(DDB) || defined(DEBUG) || defined(KGDB)
        move    ra, a3
        REG_S   ra, CALLFRAME_RA(sp)
#endif
        PTR_L   t9, P_MD_SYSCALL(t0)            # t9 = syscall
        /*
         * Turn off FPU
         */
#ifdef NOFPU
        li      t0, MIPS_SR_INT_IE
#else
        lui     t0, %hi(MIPS_SR_COP_1_BIT)
        and     t0, a1
        ori     t0, MIPS_SR_INT_IE              # turn on IEc, enable intr.
#endif
        xor     t0, a1                          # turns off the FPU & ints on
        mtc0    t0, MIPS_COP_0_STATUS           # re-enable interrupts
/*
 * Call the system call handler.
 */
        jalr    t9
         nop
/*
 * Check pending asynchronous traps.
 */
        INT_L   v0, L_MD_ASTPENDING(MIPS_CURLWP)# any pending ast?
        nop
        beqz    v0, MIPSX(user_return)          # no, skip ast processing
         nop
/*
 * We have pending asynchronous traps; all the state is already saved.
 */
        lui     ra, %hi(MIPSX(user_return))     # return directly to user return
        j       _C_LABEL(ast)
         PTR_ADDIU ra, %lo(MIPSX(user_return))  # return directly to user return
        .set    at
END(MIPSX(systemcall))

/*----------------------------------------------------------------------------
 *
 *      R3000 TLB exception handlers
 *
 *----------------------------------------------------------------------------
 */

/*----------------------------------------------------------------------------
 *
 * mipsN_kern_tlb_miss --
 *
 *      Handle a TLB miss exception from kernel mode in kernel space.
 *      The BaddVAddr, Context, and EntryHi registers contain the failed
 *      virtual address.
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------------
 */
LEAF_NOPROFILE(MIPSX(kern_tlb_miss))
        .set    noat
        _MFC0   k0, MIPS_COP_0_BAD_VADDR        # get the fault address
        PTR_LA  k1, _C_LABEL(pmap_kern_segtab)  # get address of kernel segtab
        PTR_SRL k0, SEGSHIFT - PTR_SCALESHIFT   # get segtab index (part1)
        and     k0, (NSEGPG-1) << PTR_SCALESHIFT # get segtab index (part2)
        PTR_ADDU k1, k0                         # add index to segtab addr
        PTR_L   k1, 0(k1)                       # load address of PTP
        _MFC0   k0, MIPS_COP_0_BAD_VADDR        # get the fault address
        /*
         * If there isn't a PTP for this, let trap panic for us.
         */
        beqz    k1, _C_LABEL(MIPSX(kern_gen_exception)) # full trap processing
         PTR_SRL k0, PGSHIFT - PTPSHIFT         # - delay slot -
        and     k0, (NPTEPG-1) << PTPSHIFT      # get ptp index (part2)
        PTR_ADDU k1, k0                         # add to PTP address
        INT_L   k0, 0(k1)                       # get PTE entry
        _MFC0   k1, MIPS_COP_0_EXC_PC           # get return address
        mtc0    k0, MIPS_COP_0_TLB_LOW          # save PTE entry
        and     k0, MIPS1_PG_V                  # check for valid PTE entry
        beqz    k0, _C_LABEL(MIPSX(kern_gen_exception)) # PTE invalid
         nop
        tlbwr                                   # write random TLB
        j       k1
         rfe
        .set    at
END(MIPSX(kern_tlb_miss))

#if 0
/*----------------------------------------------------------------------------
 *
 * mipsN_tlb_invalid_exception --
 *
 *      Handle a TLB modified exception.
 *      The BaddVAddr, Context, and EntryHi registers contain the failed
 *      virtual address.
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      None.
 *
 *----------------------------------------------------------------------------
 */
LEAF_NOPROFILE(MIPSX(tlb_mod_exception))
        .set    noat
        tlbp                                    # find the TLB entry
        mfc0    k0, MIPS_COP_0_TLB_LOW          # get the physical address
        mfc0    k1, MIPS_COP_0_TLB_INDEX        # check to be sure its valid
        or      k0, k0, MIPS1_TLB_DIRTY_BIT     # update TLB
        blt     k1, zero, 4f                    # not found!!!
        mtc0    k0, MIPS_COP_0_TLB_LOW
        li      k1, MIPS_KSEG0_START
        PTR_SUBU k0, k1
        srl     k0, k0, MIPS1_TLB_PHYS_PAGE_SHIFT
        PTR_L   k1, pmap_attributes             # DANGER!  DANGER!
        PTR_ADDU k0, k1
        lbu     k1, 0(k0)                       # fetch old value
        nop
        or      k1, k1, 1                       # set modified bit
        sb      k1, 0(k0)                       # save new value
        _MFC0   k0, MIPS_COP_0_EXC_PC           # get return address
        nop
        j       k0
        rfe
4:
        break   0                               # panic
        .set    at
END(MIPSX(tlb_mod_exception))
#endif

/*
 * Mark where code entered from exception handler jumptable
 * ends, for stack traceback code.
 */

        .globl  _C_LABEL(MIPSX(exceptionentry_end))
_C_LABEL(MIPSX(exceptionentry_end)):

/*--------------------------------------------------------------------------
 *
 * mipsN_tlb_get_asid --
 *
 *      Return the pid from the TLB pid reg.
 *
 *      tlb_asid_t mipsN_tlb_get_asid(void)
 *
 * Results:
 *      The current ASID.
 *
 * Side effects:
 *      None.
 *
 *--------------------------------------------------------------------------
 */
LEAF(MIPSX(tlb_get_asid))
        mfc0    v0, MIPS_COP_0_TLB_HI           # Read the hi reg value
        nop
        and     v0, MIPS1_TLB_PID               # mask out only the PID
        j       ra
         srl    v0, MIPS1_TLB_PID_SHIFT         # put PID in right spot
END(MIPSX(tlb_get_asid))

/*--------------------------------------------------------------------------
 *
 * mipsN_tlb_set_asid --
 *
 *      Write the given pid into the TLB pid reg.
 *
 *      void mipsN_tlb_set_asid(tlb_asid_t pid)
 *
 * Results:
 *      None.
 *
 * Side effects:
 *      PID set in the entry hi register.
 *
 *--------------------------------------------------------------------------
 */
LEAF(MIPSX(tlb_set_asid))
        sll     a0, MIPS1_TLB_PID_SHIFT         # put PID in right spot
        and     a0, MIPS1_TLB_PID
        mtc0    a0, MIPS_COP_0_TLB_HI           # Write the hi reg value
        j       ra
        nop
END(MIPSX(tlb_set_asid))

/*--------------------------------------------------------------------------
 *
 * mipsN_tlb_update_addr --
 *
 *      Update the TLB if highreg is found; otherwise, do_nothing
 *
 *      bool mipsN_tlb_update_addr(vaddr_t va, tlb_asid_t asid,
 *          pt_entry_t pte, bool insert);
 *
 * Results:
 *      0 if skipped, 1 if updated
 *
 * Side effects:
 *      None.
 *
 *--------------------------------------------------------------------------
 */
LEAF(MIPSX(tlb_update_addr))
        mfc0    ta0, MIPS_COP_0_STATUS          # save the status register
        mtc0    zero, MIPS_COP_0_STATUS         # disable interrupts
        nop
        mfc0    ta1, MIPS_COP_0_TLB_HI          # save current PID
        nop
        sll     a1, MIPS1_TLB_PID_SHIFT
        or      a0, a1
        mtc0    a0, MIPS_COP_0_TLB_HI           # set entryhi
        nop
        tlbp                                    # probe the existence
        mfc0    v0, MIPS_COP_0_TLB_INDEX        # see what we got
        mtc0    a2, MIPS_COP_0_TLB_LOW          # set new entrylo
        bltz    v0, 2f                          # index < 0 => !found
         nop
        tlbwi                                   # update slot found
        b       3f                              # return
         li     v0, 1                           #   and show success
2:
        beqz    a2, 3f                          # return
         li     v0, 0                           #   and show failure
        tlbwr                                   # put it in a new slot
        li      v0, 1                           # show success
3:
        mtc0    ta1, MIPS_COP_0_TLB_HI          # restore current PID
        j       ra
         mtc0   ta0, MIPS_COP_0_STATUS          # restore interrupts
END(MIPSX(tlb_update_addr))

/*--------------------------------------------------------------------------
 *
 * mipsN_tlb_read_entry --
 *
 *      Read the TLB entry.
 *
 *      void mipsN_tlb_read_entry(register_t entry, struct tlbmask *tlb)
 *
 * Results:
 *      tlb will contain the TLB entry found (tlb_lo1/tlb_mask will be 0).
 *
 *--------------------------------------------------------------------------
 */
LEAF(MIPSX(tlb_read_entry))
        mfc0    ta0, MIPS_COP_0_STATUS          # Save the status register.
        mtc0    zero, MIPS_COP_0_STATUS         # Disable interrupts
        mfc0    ta1, MIPS_COP_0_TLB_HI          # Get current PID

        sll     a0, MIPS1_TLB_INDEX_SHIFT
        mtc0    a0, MIPS_COP_0_TLB_INDEX        # Set the index register
        nop
        tlbr                                    # Read from the TLB
        mfc0    t2, MIPS_COP_0_TLB_HI           # fetch the hi entry
        mfc0    t3, MIPS_COP_0_TLB_LOW          # fetch the low entry

        mtc0    ta1, MIPS_COP_0_TLB_HI          # Restore proper PID
                                                # (before touching memory)
        mtc0    ta0, MIPS_COP_0_STATUS          # Restore the status register

        PTR_S   t2, TLBMASK_HI(a1)
        REG_S   t3, TLBMASK_LO0(a1)
        REG_S   zero, TLBMASK_LO1(a1)
        j       ra
         INT_S  zero, TLBMASK_MASK(a1)
END(MIPSX(tlb_read_entry))

/*--------------------------------------------------------------------------
 *
 * mipsX_tlb_write_entry --
 *
 *      Write the TLB entry.
 *
 *      void mipsX_tlb_write_entry(size_t entry, struct tlbmask *tlb)
 *
 * Results:
 *      None.
 *
 *--------------------------------------------------------------------------
 */
LEAF(MIPSX(tlb_write_entry))
        PTR_L   t2, TLBMASK_HI(a1)              # fetch the hi entry
        INT_L   t3, TLBMASK_LO0(a1)             # fetch the low entry
        mfc0    ta0, MIPS_COP_0_STATUS          # Save the status register.
        mtc0    zero, MIPS_COP_0_STATUS         # Disable interrupts
        mfc0    ta1, MIPS_COP_0_TLB_HI          # Get current PID

        sll     a0, MIPS1_TLB_INDEX_SHIFT
        mtc0    a0, MIPS_COP_0_TLB_INDEX        # Set the index register
        nop
        mtc0    t2, MIPS_COP_0_TLB_HI
        mtc0    t3, MIPS_COP_0_TLB_LOW

        tlbwi                                   # Write to the TLB entry

        mtc0    ta1, MIPS_COP_0_TLB_HI          # restore PID
        j       ra
         mtc0   ta0, MIPS_COP_0_STATUS          # Restore the status register
END(MIPSX(tlb_write_entry))

/*
 * void mipsN_tlb_invalidate_addr(vaddr_t va, tlb_asid_t asid)
 *
 * Invalidate a TLB entry for given virtual address if found in TLB.
 */
LEAF(MIPSX(tlb_invalidate_addr))
        mfc0    ta0, MIPS_COP_0_STATUS          # save status register
        mtc0    zero, MIPS_COP_0_STATUS         # disable interrupts
        mfc0    ta1, MIPS_COP_0_TLB_HI          # save current PID
        nop

        sll     a1, MIPS1_TLB_PID_SHIFT         # move ASID into position
        and     a1, MIPS1_TLB_PID               # make it off
        or      a0, a1                          # merge with addr
        mtc0    a0, MIPS_COP_0_TLB_HI           # look for addr & PID
        nop
        tlbp                                    # probe the entry in question
        mfc0    a0, MIPS_COP_0_TLB_INDEX        # see what we got
        li      t1, MIPS_KSEG0_START            # load invalid address
        bltz    a0, 1f                          # index < 0 then skip
         nop
        mtc0    t1, MIPS_COP_0_TLB_HI           # make entryHi invalid
        mtc0    zero, MIPS_COP_0_TLB_LOW        # zero out entryLo
        nop
        tlbwi
1:
        mtc0    ta1, MIPS_COP_0_TLB_HI          # restore PID
        j       ra
         mtc0   ta0, MIPS_COP_0_STATUS          # restore the status register
END(MIPSX(tlb_invalidate_addr))

/*
 * void mipsN_tlb_invalidate_asids(uint32_t asid_lo, uint32_t asid_hi)
 *
 * Invalidate TLB entries belonging to asids (asid_lo,asid_hi]
 * leaving entries for kernel space marked global intact.
 */
LEAF(MIPSX(tlb_invalidate_asids))
        mfc0    ta1, MIPS_COP_0_TLB_HI          # save EntryHi
        mfc0    ta0, MIPS_COP_0_STATUS          # save status register
        mtc0    zero, MIPS_COP_0_STATUS         # disable interrupts

        INT_L   t2, _C_LABEL(mips_options) + MO_NUM_TLB_ENTRIES
        li      t1, MIPS1_TLB_FIRST_RAND_ENTRY << MIPS1_TLB_INDEX_SHIFT
        li      v0, MIPS_KSEG0_START            # invalid address
        sll     t2, MIPS1_TLB_INDEX_SHIFT

        # do {} while (t1 < t2)
1:
        mtc0    t1, MIPS_COP_0_TLB_INDEX        # set index
        nop
        tlbr                                    # obtain an entry
        mfc0    t0, MIPS_COP_0_TLB_LOW
        nop
        and     t0, t0, MIPS1_PG_G              # check to see it has G bit
        bnez    t0, 2f
         nop

        mfc0    t0, MIPS_COP_0_TLB_HI           # get va and ASID
        nop
        and     t0, MIPS1_TLB_PID               # mask off ASID
        srl     t0, MIPS1_TLB_PID_SHIFT
        sltu    v1, t0, a0                      # < asid_lo
        bnez    v1, 2f                          # yes, next tlb entry
         nop
        sltu    v1, t0, a1                      # < asid_hi
        beqz    v1, 2f                          # no, next tlb entry
         nop

        mtc0    v0, MIPS_COP_0_TLB_HI           # make entryHi invalid
        mtc0    zero, MIPS_COP_0_TLB_LOW        # zero out entryLo
        nop
        tlbwi                                   # invalidate the TLB entry
2:
        addu    t1, t1, 1 << MIPS1_TLB_INDEX_SHIFT      # increment index
        bne     t1, t2, 1b
        nop

        mtc0    ta1, MIPS_COP_0_TLB_HI          # restore entryHi

        j       ra                              # new TLBpid will be set soon
         mtc0   ta0, MIPS_COP_0_STATUS          # restore status register
END(MIPSX(tlb_invalidate_asids))

/*
 * void mipsN_tlb_invalidate_all(void)
 *
 * Invalidate TLB entirely.
 */
LEAF(MIPSX(tlb_invalidate_all))
        INT_L   a0, _C_LABEL(mips_options) + MO_NUM_TLB_ENTRIES

        mfc0    ta0, MIPS_COP_0_STATUS          # save the status register.
        mtc0    zero, MIPS_COP_0_STATUS         # disable interrupts

        mfc0    ta1, MIPS_COP_0_TLB_HI          # save current PID
        li      t0, MIPS_KSEG0_START            # invalid address
        mtc0    t0, MIPS_COP_0_TLB_HI           # make entryHi invalid
        mtc0    zero, MIPS_COP_0_TLB_LOW        # zero out entryLo

        move    t0, zero
        sll     a0, MIPS1_TLB_INDEX_SHIFT

        # do {} while (t1 < a0)
1:
        mtc0    t0, MIPS_COP_0_TLB_INDEX        # set TLBindex
        addu    t0, t0, 1 << MIPS1_TLB_INDEX_SHIFT      # increment index
        bne     t0, a0, 1b
         tlbwi                                  # invalidate the entry

        mtc0    ta1, MIPS_COP_0_TLB_HI          # restore PID
        j       ra
         mtc0   ta0, MIPS_COP_0_STATUS          # restore status register
END(MIPSX(tlb_invalidate_all))

/*
 * u_int mipsN_tlb_record_asids(u_long *bitmap, uint32_t asid_max)
 *
 * Scan the random part of the TLB looking at non-global entries and
 * record each ASID in use into the bitmap.  Additionally, return the
 * number of new unique ASIDs encountered.
 */
LEAF(MIPSX(tlb_record_asids))
        mfc0    ta1, MIPS_COP_0_TLB_HI          # save EntryHi
        li      v1, MIPS1_TLB_FIRST_RAND_ENTRY << MIPS1_TLB_INDEX_SHIFT
        INT_L   a3, _C_LABEL(mips_options) + MO_NUM_TLB_ENTRIES
        move    ta2, zero
        li      ta3, 1
        sll     a3, MIPS1_TLB_INDEX_SHIFT

        mfc0    ta0, MIPS_COP_0_STATUS          # save status register
        mtc0    zero, MIPS_COP_0_STATUS         # disable interrupts

        move    v0, zero                        # start at zero ASIDs

        # do {} while (v1 < ta1)
1:
        mtc0    v1, MIPS_COP_0_TLB_INDEX        # set index
        nop
        tlbr                                    # obtain an entry
        mfc0    t0, MIPS_COP_0_TLB_LOW
        nop
        and     t0, MIPS1_PG_G                  # check to see it has G bit
        bnez    t0, 4f
         nop

        mfc0    t0, MIPS_COP_0_TLB_HI           # get va and ASID
        nop
        and     t0, MIPS1_TLB_PID
        srl     t0, MIPS1_TLB_PID_SHIFT         # shift to low bits
        bgt     t0, a1, 4f                      # > ASID max? skip
         nop

        srl     a2, t0, 3 + LONG_SCALESHIFT     # drop low 5 bits
        sll     a2, LONG_SCALESHIFT             # make an index for the bitmap
        sllv    t0, ta3, t0                     # t0 is mask (ta3 == 1)

        PTR_ADDU a2, a0                         # index into the bitmap
        beq     a2, ta2, 3f                     # is the desired cell loaded?
         nop                                    #   yes, don't reload it
        beqz    ta2, 2f                         # have we ever loaded it?
         nop                                    #   nope, so don't save it

        LONG_S  t2, 0(ta2)                      # save the updated value.
2:
        LONG_L  t2, 0(a2)                       # and load it
         move   ta2, a2                         # remember the new cell's addr
3:
        and     t1, t2, t0                      # see if this asid was recorded
        sltu    t1, t1, ta3                     # t1 = t1 < 1 (aka t1 == 0)
        addu    v0, t1                          # v0 += t1
        or      t2, t0                          # or in the new ASID bits

4:
        addu    v1, 1 << MIPS1_TLB_INDEX_SHIFT  # increment TLB entry #
        bne     v1, a3, 1b                      # keep lookup if not limit
         nop

        beqz    ta2, 5f                         # do we have a cell to write?
         nop                                    #   nope, nothing.

        LONG_S  t2, 0(ta2)                      # save the updated value.
5:
        mtc0    ta1, MIPS_COP_0_TLB_HI          # restore entryHi

        j       ra                              # new TLBpid will be set soon
         mtc0   ta0, MIPS_COP_0_STATUS          # restore status register
END(MIPSX(tlb_record_asids))

/*----------------------------------------------------------------------------
 *
 *      R3000 trampolines and context resume
 *
 *----------------------------------------------------------------------------
 */

/*----------------------------------------------------------------------------
 *
 * mipsN_lwp_trampoline
 *
 * Special arrangement for a process about to go user mode right after
 * fork() system call.  When the first CPU tick is scheduled to run the
 * forked child, it starts running from here.  Then, a service function
 * is called with one argument supplied to complete final preparations,
 * and the process returns to user mode as if the fork() system call is
 * handled in a normal way.  No need to save any registers although this
 * calls another.
 *----------------------------------------------------------------------------
 */
LEAF(MIPSX(lwp_trampoline))
        PTR_ADDU sp, -CALLFRAME_SIZ

        # Call lwp_startup(), with args from cpu_switchto()/cpu_lwp_fork()
        move    a0, v0
        jal     _C_LABEL(lwp_startup)
         move   a1, MIPS_CURLWP

        # Call the routine specified by cpu_lwp_fork()
        jalr    s0
         move   a0, s1

        # Return to user (won't happen if a kernel thread)
        .set    noat
MIPSX(user_return):
        REG_L   s0, CALLFRAME_SIZ+TF_REG_S0(sp)         # $16
        REG_L   s1, CALLFRAME_SIZ+TF_REG_S1(sp)         # $17
        REG_L   s2, CALLFRAME_SIZ+TF_REG_S2(sp)         # $18
        REG_L   s3, CALLFRAME_SIZ+TF_REG_S3(sp)         # $19
        REG_L   s4, CALLFRAME_SIZ+TF_REG_S4(sp)         # $20
        REG_L   s5, CALLFRAME_SIZ+TF_REG_S5(sp)         # $21
        REG_L   s6, CALLFRAME_SIZ+TF_REG_S6(sp)         # $22
        REG_L   s7, CALLFRAME_SIZ+TF_REG_S7(sp)         # $23
        REG_L   s8, CALLFRAME_SIZ+TF_REG_S8(sp)         # $30
MIPSX(user_intr_return):
        REG_L   a0, CALLFRAME_SIZ+TF_REG_SR(sp)
        REG_L   t0, CALLFRAME_SIZ+TF_REG_MULLO(sp)
        REG_L   t1, CALLFRAME_SIZ+TF_REG_MULHI(sp)
        mtc0    a0, MIPS_COP_0_STATUS   # this should disable interrupts
        mtlo    t0
        mthi    t1
        move    k1, sp
        REG_L   AT, TF_BASE+TF_REG_AST(sp)
        REG_L   k0, CALLFRAME_SIZ+TF_REG_EPC(k1)
        REG_L   AT, CALLFRAME_SIZ+TF_REG_AST(k1)
        REG_L   v0, CALLFRAME_SIZ+TF_REG_V0(k1)
        REG_L   v1, CALLFRAME_SIZ+TF_REG_V1(k1)
        REG_L   a0, CALLFRAME_SIZ+TF_REG_A0(k1)
        REG_L   a1, CALLFRAME_SIZ+TF_REG_A1(k1)
        REG_L   a2, CALLFRAME_SIZ+TF_REG_A2(k1)
        REG_L   a3, CALLFRAME_SIZ+TF_REG_A3(k1)
        REG_L   t0, CALLFRAME_SIZ+TF_REG_T0(k1)
        REG_L   t1, CALLFRAME_SIZ+TF_REG_T1(k1)
        REG_L   t2, CALLFRAME_SIZ+TF_REG_T2(k1)
        REG_L   t3, CALLFRAME_SIZ+TF_REG_T3(k1)
        REG_L   ta0, CALLFRAME_SIZ+TF_REG_TA0(k1)
        REG_L   ta1, CALLFRAME_SIZ+TF_REG_TA1(k1)
        REG_L   ta2, CALLFRAME_SIZ+TF_REG_TA2(k1)
        REG_L   ta3, CALLFRAME_SIZ+TF_REG_TA3(k1)
        REG_L   t8, CALLFRAME_SIZ+TF_REG_T8(k1)
        REG_L   t9, CALLFRAME_SIZ+TF_REG_T9(k1)
        REG_L   gp, CALLFRAME_SIZ+TF_REG_GP(k1)
        REG_L   ra, CALLFRAME_SIZ+TF_REG_RA(k1)
        REG_L   sp, CALLFRAME_SIZ+TF_REG_SP(k1)
        nop
        j       k0
        rfe
        .set    at
END(MIPSX(lwp_trampoline))

/*
 * void mipsN_cpu_switch_resume(struct lwp *newlwp)
 *
 * Wiredown the USPACE of newproc with TLB entry#0 and #1.  Check
 * if target USPACE is already referred by any TLB entry before
 * doing that, and make sure TBIS(them) in the case.
 */
LEAF_NOPROFILE(MIPSX(cpu_switch_resume))
        INT_L   a1, L_MD_UPTE_0(a0)             # a1 = upte[0]
        INT_L   a2, L_MD_UPTE_1(a0)             # a2 = upte[1]
        PTR_L   s0, L_PCB(a0)                   # va = l->l_addr
        li      s2, VM_MIN_KERNEL_ADDRESS
        blt     s0, s2, resume
        nop

        mfc0    t3, MIPS_COP_0_TLB_HI           # save PID
        nop
        mtc0    s0, MIPS_COP_0_TLB_HI           # VPN = va
        nop
        tlbp                                    # probe 1st VPN
        mfc0    s1, MIPS_COP_0_TLB_INDEX
        nop
        bltz    s1, entry0set
        li      s1, MIPS_KSEG0_START            # found, then
        mtc0    s1, MIPS_COP_0_TLB_HI
        mtc0    zero, MIPS_COP_0_TLB_LOW
        nop
        tlbwi                                   # TBIS(va)
        nop
        mtc0    s0, MIPS_COP_0_TLB_HI           # set 1st VPN again
entry0set:
        mtc0    zero, MIPS_COP_0_TLB_INDEX      # TLB index #0
        ori     a1, a1, MIPS1_PG_G
        mtc0    a1, MIPS_COP_0_TLB_LOW          # 1st PFN w/ PG_G
        nop
        tlbwi                                   # set TLB entry #0

        addu    s0, s0, PAGE_SIZE
        mtc0    s0, MIPS_COP_0_TLB_HI           # VPN = va+PAGE_SIZE
        nop
        tlbp                                    # probe 2nd VPN
        mfc0    s1, MIPS_COP_0_TLB_INDEX
        nop
        bltz    s1, entry1set
        li      s1, MIPS_KSEG0_START            # found, then
        mtc0    s1, MIPS_COP_0_TLB_HI
        mtc0    zero, MIPS_COP_0_TLB_LOW
        nop
        tlbwi                                   # TBIS(va+PAGE_SIZE)
        nop
        mtc0    s0, MIPS_COP_0_TLB_HI           # set 2nd VPN again
entry1set:
        li      s1, 1 << MIPS1_TLB_INDEX_SHIFT
        mtc0    s1, MIPS_COP_0_TLB_INDEX        # TLB index #1
        ori     a2, a2, MIPS1_PG_G
        mtc0    a2, MIPS_COP_0_TLB_LOW          # 2nd PFN w/ PG_G
        nop
        tlbwi                                   # set TLB entry #1
        nop
        mfc0    t3, MIPS_COP_0_TLB_HI           # restore PID

resume:
        j       ra
        nop
END(MIPSX(cpu_switch_resume))

/*----------------------------------------------------------------------------
 *
 *      R3000 cache sizing and flushing code.
 *
 *----------------------------------------------------------------------------
 */
#ifndef ENABLE_MIPS_TX3900
/*
 * void mipsN_wbflush(void)
 *
 * Drain processor's write buffer, normally used to ensure any I/O
 * register write operations are done before subsequent manipulations.
 *
 * Some hardware implementations have a WB chip independent from CPU
 * core, and CU0 (Coprocessor Usability #0) bit of CP0 status register
 * is wired to indicate writebuffer condition.  This code does busy-loop
 * while CU0 bit indicates false condition.
 *
 * For other hardware which have the writebuffer logic is implemented
 * in a system controller ASIC chip, wbflush operation would done
 * differently.
 */
LEAF(MIPSX(wbflush))
        nop
        nop
        nop
        nop
1:      bc0f    1b
        nop
        j       ra
        nop
END(MIPSX(wbflush))
#else /* !ENABLE_MIPS_TX3900 */
/*
 *      The differences between R3900 and R3000.
 *      1. Cache system
 *              Physical-index physical-tag
 *              fixed line-size
 *              refil-size 4/8/16/32 words (set in config register)
 *              TX3912
 *                     Write-through
 *                     I-cache 4KB/16B direct mapped (256line)
 *                     D-cache 1KB/4B 2-way sa (128line)
 *                     Cache snoop
 *              TX3922
 *                     Write-through/write-back (set in config register)
 *                     I-cache 16KB/16B 2-way sa
 *                     D-cache 8KB/16B 2-way sa
 *                     Cache snoop
 *
 *      2. Coprocessor1
 *      2.1     cache operation.
 *              R3900 uses MIPSIII cache op like method.
 *      2.2     R3900 specific CP0 register.
 *              (mips/include/r3900regs.h overrides cpuregs.h)
 *      2.3     # of TLB entries
 *              TX3912 32 entries
 *              TX3922 64 entries
 *
 *      3. System address map
 *              kseg2 0xff000000-0xfffeffff is reserved.
 *              (mips/include/vmparam.h)
 *
 *  + If defined both MIPS1 and ENABLE_MIPS_TX3900, it generates kernel for
 * R3900. If defined MIPS1 only, No R3900 feature include.
 *  + R3920 core has write-back mode. but it is always disabled in NetBSD.
 */

LEAF_NOPROFILE(tx3900_cp0_config_read)
        mfc0    v0, R3900_COP_0_CONFIG
        j       ra
         nop
END(tx3900_cp0_config_read)

LEAF(MIPSX(wbflush))
        .set push
        .set mips2
        sync
        .set pop
        j       ra
        nop
END(MIPSX(wbflush))
#endif /* !ENABLE_MIPS_TX3900 */

        .rdata

        .globl _C_LABEL(MIPSX(locore_vec))
_C_LABEL(MIPSX(locore_vec)):
        PTR_WORD _C_LABEL(MIPSX(cpu_switch_resume))
        PTR_WORD _C_LABEL(MIPSX(lwp_trampoline))
        PTR_WORD _C_LABEL(MIPSX(wbflush))               # wbflush
        PTR_WORD _C_LABEL(MIPSX(tlb_get_asid))
        PTR_WORD _C_LABEL(MIPSX(tlb_set_asid))
        PTR_WORD _C_LABEL(MIPSX(tlb_invalidate_asids))
        PTR_WORD _C_LABEL(MIPSX(tlb_invalidate_addr))
        PTR_WORD _C_LABEL(nullop)                       # tlb_invalidate_globals
        PTR_WORD _C_LABEL(MIPSX(tlb_invalidate_all))
        PTR_WORD _C_LABEL(MIPSX(tlb_record_asids))
        PTR_WORD _C_LABEL(MIPSX(tlb_update_addr))
        PTR_WORD _C_LABEL(MIPSX(tlb_read_entry))
        PTR_WORD _C_LABEL(MIPSX(tlb_write_entry))

        .globl _C_LABEL(MIPSX(locoresw))
_C_LABEL(MIPSX(locoresw)):
        PTR_WORD _C_LABEL(MIPSX(wbflush))               # lsw_wbflush
        PTR_WORD _C_LABEL(nullop)                       # lsw_cpu_idle
        PTR_WORD _C_LABEL(nullop)                       # lsw_send_ipi
        PTR_WORD _C_LABEL(nullop)                       # lsw_cpu_offline_md
        PTR_WORD _C_LABEL(nullop)                       # lsw_cpu_init
        PTR_WORD _C_LABEL(nullop)                       # lsw_cpu_run
        PTR_WORD _C_LABEL(nullop)                       # lsw_bus_error

MIPSX(excpt_sw):
        ####
        #### The kernel exception handlers.
        ####
        PTR_WORD _C_LABEL(MIPSX(kern_intr))     # 0 external interrupt
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 1 TLB modification
        PTR_WORD _C_LABEL(MIPSX(kern_tlb_miss)) # 2 TLB miss (LW/I-fetch)
        PTR_WORD _C_LABEL(MIPSX(kern_tlb_miss)) # 3 TLB miss (SW)
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 4 address error (LW/I-fetch)
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 5 address error (SW)
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 6 bus error (I-fetch)
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 7 bus error (load or store)
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 8 system call
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 9 breakpoint
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 10 reserved instruction
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 11 coprocessor unusable
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 12 arithmetic overflow
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 13 r3k reserved
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 14 r3k reserved
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 15 r3k reserved
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 16 never happens w/ MIPS1
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 17 never happens w/ MIPS1
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 18 never happens w/ MIPS1
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 19 never happens w/ MIPS1
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 20 never happens w/ MIPS1
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 21 never happens w/ MIPS1
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 22 never happens w/ MIPS1
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 23 never happens w/ MIPS1
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 24 never happens w/ MIPS1
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 25 never happens w/ MIPS1
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 26 never happens w/ MIPS1
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 27 never happens w/ MIPS1
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 28 never happens w/ MIPS1
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 29 never happens w/ MIPS1
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 30 never happens w/ MIPS1
        PTR_WORD _C_LABEL(MIPSX(kern_gen_exception))# 31 never happens w/ MIPS1
        #####
        ##### The user exception handlers.
        #####
        PTR_WORD _C_LABEL(MIPSX(user_intr))     #  0
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))#  1
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))#  2
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))#  3
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))#  4
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))#  5
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))#  6
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))#  7
        PTR_WORD _C_LABEL(MIPSX(systemcall))    #  8
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))#  9
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 10
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 11
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 12
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 13
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 14
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 15
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 16
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 17
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 18
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 19
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 20
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 21
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 22
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 23
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 24
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 25
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 26
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 27
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 28
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 29
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 20
        PTR_WORD _C_LABEL(MIPSX(user_gen_exception))# 31