/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

#include "assym.h"
#include <sys/asm_linkage.h>
#include <sys/privregs.h>
#include <sys/sun4asi.h>
#include <sys/machasi.h>
#include <sys/hypervisor_api.h>
#include <sys/machtrap.h>
#include <sys/machthread.h>
#include <sys/machbrand.h>
#include <sys/pcb.h>
#include <sys/pte.h>
#include <sys/mmu.h>
#include <sys/machpcb.h>
#include <sys/async.h>
#include <sys/intreg.h>
#include <sys/scb.h>
#include <sys/psr_compat.h>
#include <sys/syscall.h>
#include <sys/machparam.h>
#include <sys/traptrace.h>
#include <vm/hat_sfmmu.h>
#include <sys/archsystm.h>
#include <sys/utrap.h>
#include <sys/clock.h>
#include <sys/intr.h>
#include <sys/fpu/fpu_simulator.h>
#include <vm/seg_spt.h>

/*
 * WARNING: If you add a fast trap handler which can be invoked by a
 * non-privileged user, you may have to use the FAST_TRAP_DONE macro
 * instead of "done" instruction to return back to the user mode. See
 * comments for the "fast_trap_done" entry point for more information.
 *
 * An alternate FAST_TRAP_DONE_CHK_INTR macro should be used for the
 * cases where you always want to process any pending interrupts before
 * returning back to the user mode.
 */
#define FAST_TRAP_DONE          \
        ba,a    fast_trap_done

#define FAST_TRAP_DONE_CHK_INTR \
        ba,a    fast_trap_done_chk_intr

/*
 * SPARC V9 Trap Table
 *
 * Most of the trap handlers are made from common building
 * blocks, and some are instantiated multiple times within
 * the trap table. So, I build a bunch of macros, then
 * populate the table using only the macros.
 *
 * Many macros branch to sys_trap.  Its calling convention is:
 *      %g1             kernel trap handler
 *      %g2, %g3        args for above
 *      %g4             desire %pil
 */

#ifdef  TRAPTRACE

/*
 * Tracing macro. Adds two instructions if TRAPTRACE is defined.
 */
#define TT_TRACE(label)         \
        ba      label           ;\
        rd      %pc, %g7
#define TT_TRACE_INS    2

#define TT_TRACE_L(label)       \
        ba      label           ;\
        rd      %pc, %l4        ;\
        clr     %l4
#define TT_TRACE_L_INS  3

#else

#define TT_TRACE(label)
#define TT_TRACE_INS    0

#define TT_TRACE_L(label)
#define TT_TRACE_L_INS  0

#endif

/*
 * This first set are funneled to trap() with %tt as the type.
 * Trap will then either panic or send the user a signal.
 */
/*
 * NOT is used for traps that just shouldn't happen.
 * It comes in both single and quadruple flavors.
 */
        .global trap
#define NOT                     \
        TT_TRACE(trace_gen)     ;\
        set     trap, %g1       ;\
        rdpr    %tt, %g3        ;\
        ba,pt   %xcc, sys_trap  ;\
        sub     %g0, 1, %g4     ;\
        .align  32
#define NOT4    NOT; NOT; NOT; NOT

#define NOTP                            \
        TT_TRACE(trace_gen)             ;\
        ba,pt   %xcc, ptl1_panic        ;\
          mov   PTL1_BAD_TRAP, %g1      ;\
        .align  32
#define NOTP4   NOTP; NOTP; NOTP; NOTP


/*
 * BAD is used for trap vectors we don't have a kernel
 * handler for.
 * It also comes in single and quadruple versions.
 */
#define BAD     NOT
#define BAD4    NOT4

#define DONE                    \
        done;                   \
        .align  32

/*
 * TRAP vectors to the trap() function.
 * It's main use is for user errors.
 */
        .global trap
#define TRAP(arg)               \
        TT_TRACE(trace_gen)     ;\
        set     trap, %g1       ;\
        mov     arg, %g3        ;\
        ba,pt   %xcc, sys_trap  ;\
        sub     %g0, 1, %g4     ;\
        .align  32

/*
 * SYSCALL is used for unsupported syscall interfaces (with 'which'
 * set to 'nosys') and legacy support of old SunOS 4.x syscalls (with
 * 'which' set to 'syscall_trap32').
 *
 * The SYSCALL_TRAP* macros are used for syscall entry points.
 * SYSCALL_TRAP is used to support LP64 syscalls and SYSCALL_TRAP32
 * is used to support ILP32.  Each macro can only be used once
 * since they each define a symbol.  The symbols are used as hot patch
 * points by the brand infrastructure to dynamically enable and disable
 * brand syscall interposition.  See the comments around BRAND_CALLBACK
 * and brand_plat_interposition_enable() for more information.
 */
#define SYSCALL_NOTT(which)             \
        set     (which), %g1            ;\
        ba,pt   %xcc, user_trap         ;\
        sub     %g0, 1, %g4             ;\
        .align  32

#define SYSCALL(which)                  \
        TT_TRACE(trace_gen)             ;\
        SYSCALL_NOTT(which)

#define SYSCALL_TRAP32                          \
        TT_TRACE(trace_gen)                     ;\
        ALTENTRY(syscall_trap32_patch_point)    \
        SYSCALL_NOTT(syscall_trap32)

#define SYSCALL_TRAP                            \
        TT_TRACE(trace_gen)                     ;\
        ALTENTRY(syscall_trap_patch_point)      \
        SYSCALL_NOTT(syscall_trap)

/*
 * GOTO just jumps to a label.
 * It's used for things that can be fixed without going thru sys_trap.
 */
#define GOTO(label)             \
        .global label           ;\
        ba,a    label           ;\
        .empty                  ;\
        .align  32

/*
 * GOTO_TT just jumps to a label.
 * correctable ECC error traps at  level 0 and 1 will use this macro.
 * It's used for things that can be fixed without going thru sys_trap.
 */
#define GOTO_TT(label, ttlabel)         \
        .global label           ;\
        TT_TRACE(ttlabel)       ;\
        ba,a    label           ;\
        .empty                  ;\
        .align  32

/*
 * Privileged traps
 * Takes breakpoint if privileged, calls trap() if not.
 */
#define PRIV(label)                     \
        rdpr    %tstate, %g1            ;\
        btst    TSTATE_PRIV, %g1        ;\
        bnz     label                   ;\
        rdpr    %tt, %g3                ;\
        set     trap, %g1               ;\
        ba,pt   %xcc, sys_trap          ;\
        sub     %g0, 1, %g4             ;\
        .align  32


/*
 * DTrace traps.
 */
#define DTRACE_PID                      \
        .global dtrace_pid_probe                                ;\
        set     dtrace_pid_probe, %g1                           ;\
        ba,pt   %xcc, user_trap                                 ;\
        sub     %g0, 1, %g4                                     ;\
        .align  32

#define DTRACE_RETURN                   \
        .global dtrace_return_probe                             ;\
        set     dtrace_return_probe, %g1                        ;\
        ba,pt   %xcc, user_trap                                 ;\
        sub     %g0, 1, %g4                                     ;\
        .align  32

/*
 * REGISTER WINDOW MANAGEMENT MACROS
 */

/*
 * various convenient units of padding
 */
#define SKIP(n) .skip 4*(n)

/*
 * CLEAN_WINDOW is the simple handler for cleaning a register window.
 */
#define CLEAN_WINDOW                                            \
        TT_TRACE_L(trace_win)                                   ;\
        rdpr %cleanwin, %l0; inc %l0; wrpr %l0, %cleanwin       ;\
        clr %l0; clr %l1; clr %l2; clr %l3                      ;\
        clr %l4; clr %l5; clr %l6; clr %l7                      ;\
        clr %o0; clr %o1; clr %o2; clr %o3                      ;\
        clr %o4; clr %o5; clr %o6; clr %o7                      ;\
        retry; .align 128

/*
 * If we get an unresolved tlb miss while in a window handler, the fault
 * handler will resume execution at the last instruction of the window
 * hander, instead of delivering the fault to the kernel.  Spill handlers
 * use this to spill windows into the wbuf.
 *
 * The mixed handler works by checking %sp, and branching to the correct
 * handler.  This is done by branching back to label 1: for 32b frames,
 * or label 2: for 64b frames; which implies the handler order is: 32b,
 * 64b, mixed.  The 1: and 2: labels are offset into the routines to
 * allow the branchs' delay slots to contain useful instructions.
 */

/*
 * SPILL_32bit spills a 32-bit-wide kernel register window.  It
 * assumes that the kernel context and the nucleus context are the
 * same.  The stack pointer is required to be eight-byte aligned even
 * though this code only needs it to be four-byte aligned.
 */
#define SPILL_32bit(tail)                                       \
        srl     %sp, 0, %sp                                     ;\
1:      st      %l0, [%sp + 0]                                  ;\
        st      %l1, [%sp + 4]                                  ;\
        st      %l2, [%sp + 8]                                  ;\
        st      %l3, [%sp + 12]                                 ;\
        st      %l4, [%sp + 16]                                 ;\
        st      %l5, [%sp + 20]                                 ;\
        st      %l6, [%sp + 24]                                 ;\
        st      %l7, [%sp + 28]                                 ;\
        st      %i0, [%sp + 32]                                 ;\
        st      %i1, [%sp + 36]                                 ;\
        st      %i2, [%sp + 40]                                 ;\
        st      %i3, [%sp + 44]                                 ;\
        st      %i4, [%sp + 48]                                 ;\
        st      %i5, [%sp + 52]                                 ;\
        st      %i6, [%sp + 56]                                 ;\
        st      %i7, [%sp + 60]                                 ;\
        TT_TRACE_L(trace_win)                                   ;\
        saved                                                   ;\
        retry                                                   ;\
        SKIP(31-19-TT_TRACE_L_INS)                              ;\
        ba,a,pt %xcc, fault_32bit_##tail                        ;\
        .empty

/*
 * SPILL_32bit_asi spills a 32-bit-wide register window into a 32-bit
 * wide address space via the designated asi.  It is used to spill
 * non-kernel windows.  The stack pointer is required to be eight-byte
 * aligned even though this code only needs it to be four-byte
 * aligned.
 */
#define SPILL_32bit_asi(asi_num, tail)                          \
        srl     %sp, 0, %sp                                     ;\
1:      sta     %l0, [%sp + %g0]asi_num                         ;\
        mov     4, %g1                                          ;\
        sta     %l1, [%sp + %g1]asi_num                         ;\
        mov     8, %g2                                          ;\
        sta     %l2, [%sp + %g2]asi_num                         ;\
        mov     12, %g3                                         ;\
        sta     %l3, [%sp + %g3]asi_num                         ;\
        add     %sp, 16, %g4                                    ;\
        sta     %l4, [%g4 + %g0]asi_num                         ;\
        sta     %l5, [%g4 + %g1]asi_num                         ;\
        sta     %l6, [%g4 + %g2]asi_num                         ;\
        sta     %l7, [%g4 + %g3]asi_num                         ;\
        add     %g4, 16, %g4                                    ;\
        sta     %i0, [%g4 + %g0]asi_num                         ;\
        sta     %i1, [%g4 + %g1]asi_num                         ;\
        sta     %i2, [%g4 + %g2]asi_num                         ;\
        sta     %i3, [%g4 + %g3]asi_num                         ;\
        add     %g4, 16, %g4                                    ;\
        sta     %i4, [%g4 + %g0]asi_num                         ;\
        sta     %i5, [%g4 + %g1]asi_num                         ;\
        sta     %i6, [%g4 + %g2]asi_num                         ;\
        sta     %i7, [%g4 + %g3]asi_num                         ;\
        TT_TRACE_L(trace_win)                                   ;\
        saved                                                   ;\
        retry                                                   ;\
        SKIP(31-25-TT_TRACE_L_INS)                              ;\
        ba,a,pt %xcc, fault_32bit_##tail                        ;\
        .empty

#define SPILL_32bit_tt1(asi_num, tail)                          \
        ba,a,pt %xcc, fault_32bit_##tail                        ;\
        .empty                                                  ;\
        .align 128


/*
 * FILL_32bit fills a 32-bit-wide kernel register window.  It assumes
 * that the kernel context and the nucleus context are the same.  The
 * stack pointer is required to be eight-byte aligned even though this
 * code only needs it to be four-byte aligned.
 */
#define FILL_32bit(tail)                                        \
        srl     %sp, 0, %sp                                     ;\
1:      TT_TRACE_L(trace_win)                                   ;\
        ld      [%sp + 0], %l0                                  ;\
        ld      [%sp + 4], %l1                                  ;\
        ld      [%sp + 8], %l2                                  ;\
        ld      [%sp + 12], %l3                                 ;\
        ld      [%sp + 16], %l4                                 ;\
        ld      [%sp + 20], %l5                                 ;\
        ld      [%sp + 24], %l6                                 ;\
        ld      [%sp + 28], %l7                                 ;\
        ld      [%sp + 32], %i0                                 ;\
        ld      [%sp + 36], %i1                                 ;\
        ld      [%sp + 40], %i2                                 ;\
        ld      [%sp + 44], %i3                                 ;\
        ld      [%sp + 48], %i4                                 ;\
        ld      [%sp + 52], %i5                                 ;\
        ld      [%sp + 56], %i6                                 ;\
        ld      [%sp + 60], %i7                                 ;\
        restored                                                ;\
        retry                                                   ;\
        SKIP(31-19-TT_TRACE_L_INS)                              ;\
        ba,a,pt %xcc, fault_32bit_##tail                        ;\
        .empty

/*
 * FILL_32bit_asi fills a 32-bit-wide register window from a 32-bit
 * wide address space via the designated asi.  It is used to fill
 * non-kernel windows.  The stack pointer is required to be eight-byte
 * aligned even though this code only needs it to be four-byte
 * aligned.
 */
#define FILL_32bit_asi(asi_num, tail)                           \
        srl     %sp, 0, %sp                                     ;\
1:      TT_TRACE_L(trace_win)                                   ;\
        mov     4, %g1                                          ;\
        lda     [%sp + %g0]asi_num, %l0                         ;\
        mov     8, %g2                                          ;\
        lda     [%sp + %g1]asi_num, %l1                         ;\
        mov     12, %g3                                         ;\
        lda     [%sp + %g2]asi_num, %l2                         ;\
        lda     [%sp + %g3]asi_num, %l3                         ;\
        add     %sp, 16, %g4                                    ;\
        lda     [%g4 + %g0]asi_num, %l4                         ;\
        lda     [%g4 + %g1]asi_num, %l5                         ;\
        lda     [%g4 + %g2]asi_num, %l6                         ;\
        lda     [%g4 + %g3]asi_num, %l7                         ;\
        add     %g4, 16, %g4                                    ;\
        lda     [%g4 + %g0]asi_num, %i0                         ;\
        lda     [%g4 + %g1]asi_num, %i1                         ;\
        lda     [%g4 + %g2]asi_num, %i2                         ;\
        lda     [%g4 + %g3]asi_num, %i3                         ;\
        add     %g4, 16, %g4                                    ;\
        lda     [%g4 + %g0]asi_num, %i4                         ;\
        lda     [%g4 + %g1]asi_num, %i5                         ;\
        lda     [%g4 + %g2]asi_num, %i6                         ;\
        lda     [%g4 + %g3]asi_num, %i7                         ;\
        restored                                                ;\
        retry                                                   ;\
        SKIP(31-25-TT_TRACE_L_INS)                              ;\
        ba,a,pt %xcc, fault_32bit_##tail                        ;\
        .empty


/*
 * SPILL_64bit spills a 64-bit-wide kernel register window.  It
 * assumes that the kernel context and the nucleus context are the
 * same.  The stack pointer is required to be eight-byte aligned.
 */
#define SPILL_64bit(tail)                                       \
2:      stx     %l0, [%sp + V9BIAS64 + 0]                       ;\
        stx     %l1, [%sp + V9BIAS64 + 8]                       ;\
        stx     %l2, [%sp + V9BIAS64 + 16]                      ;\
        stx     %l3, [%sp + V9BIAS64 + 24]                      ;\
        stx     %l4, [%sp + V9BIAS64 + 32]                      ;\
        stx     %l5, [%sp + V9BIAS64 + 40]                      ;\
        stx     %l6, [%sp + V9BIAS64 + 48]                      ;\
        stx     %l7, [%sp + V9BIAS64 + 56]                      ;\
        stx     %i0, [%sp + V9BIAS64 + 64]                      ;\
        stx     %i1, [%sp + V9BIAS64 + 72]                      ;\
        stx     %i2, [%sp + V9BIAS64 + 80]                      ;\
        stx     %i3, [%sp + V9BIAS64 + 88]                      ;\
        stx     %i4, [%sp + V9BIAS64 + 96]                      ;\
        stx     %i5, [%sp + V9BIAS64 + 104]                     ;\
        stx     %i6, [%sp + V9BIAS64 + 112]                     ;\
        stx     %i7, [%sp + V9BIAS64 + 120]                     ;\
        TT_TRACE_L(trace_win)                                   ;\
        saved                                                   ;\
        retry                                                   ;\
        SKIP(31-18-TT_TRACE_L_INS)                              ;\
        ba,a,pt %xcc, fault_64bit_##tail                        ;\
        .empty

#define SPILL_64bit_ktt1(tail)                          \
        ba,a,pt %xcc, fault_64bit_##tail                        ;\
        .empty                                                  ;\
        .align 128

#define SPILL_mixed_ktt1(tail)                          \
        btst    1, %sp                                          ;\
        bz,a,pt %xcc, fault_32bit_##tail                        ;\
        srl     %sp, 0, %sp                                     ;\
        ba,a,pt %xcc, fault_64bit_##tail                        ;\
        .empty                                                  ;\
        .align 128

/*
 * SPILL_64bit_asi spills a 64-bit-wide register window into a 64-bit
 * wide address space via the designated asi.  It is used to spill
 * non-kernel windows.  The stack pointer is required to be eight-byte
 * aligned.
 */
#define SPILL_64bit_asi(asi_num, tail)                          \
        mov     0 + V9BIAS64, %g1                               ;\
2:      stxa    %l0, [%sp + %g1]asi_num                         ;\
        mov     8 + V9BIAS64, %g2                               ;\
        stxa    %l1, [%sp + %g2]asi_num                         ;\
        mov     16 + V9BIAS64, %g3                              ;\
        stxa    %l2, [%sp + %g3]asi_num                         ;\
        mov     24 + V9BIAS64, %g4                              ;\
        stxa    %l3, [%sp + %g4]asi_num                         ;\
        add     %sp, 32, %g5                                    ;\
        stxa    %l4, [%g5 + %g1]asi_num                         ;\
        stxa    %l5, [%g5 + %g2]asi_num                         ;\
        stxa    %l6, [%g5 + %g3]asi_num                         ;\
        stxa    %l7, [%g5 + %g4]asi_num                         ;\
        add     %g5, 32, %g5                                    ;\
        stxa    %i0, [%g5 + %g1]asi_num                         ;\
        stxa    %i1, [%g5 + %g2]asi_num                         ;\
        stxa    %i2, [%g5 + %g3]asi_num                         ;\
        stxa    %i3, [%g5 + %g4]asi_num                         ;\
        add     %g5, 32, %g5                                    ;\
        stxa    %i4, [%g5 + %g1]asi_num                         ;\
        stxa    %i5, [%g5 + %g2]asi_num                         ;\
        stxa    %i6, [%g5 + %g3]asi_num                         ;\
        stxa    %i7, [%g5 + %g4]asi_num                         ;\
        TT_TRACE_L(trace_win)                                   ;\
        saved                                                   ;\
        retry                                                   ;\
        SKIP(31-25-TT_TRACE_L_INS)                              ;\
        ba,a,pt %xcc, fault_64bit_##tail                        ;\
        .empty

#define SPILL_64bit_tt1(asi_num, tail)                          \
        ba,a,pt %xcc, fault_64bit_##tail                        ;\
        .empty                                                  ;\
        .align 128

/*
 * FILL_64bit fills a 64-bit-wide kernel register window.  It assumes
 * that the kernel context and the nucleus context are the same.  The
 * stack pointer is required to be eight-byte aligned.
 */
#define FILL_64bit(tail)                                        \
2:      TT_TRACE_L(trace_win)                                   ;\
        ldx     [%sp + V9BIAS64 + 0], %l0                       ;\
        ldx     [%sp + V9BIAS64 + 8], %l1                       ;\
        ldx     [%sp + V9BIAS64 + 16], %l2                      ;\
        ldx     [%sp + V9BIAS64 + 24], %l3                      ;\
        ldx     [%sp + V9BIAS64 + 32], %l4                      ;\
        ldx     [%sp + V9BIAS64 + 40], %l5                      ;\
        ldx     [%sp + V9BIAS64 + 48], %l6                      ;\
        ldx     [%sp + V9BIAS64 + 56], %l7                      ;\
        ldx     [%sp + V9BIAS64 + 64], %i0                      ;\
        ldx     [%sp + V9BIAS64 + 72], %i1                      ;\
        ldx     [%sp + V9BIAS64 + 80], %i2                      ;\
        ldx     [%sp + V9BIAS64 + 88], %i3                      ;\
        ldx     [%sp + V9BIAS64 + 96], %i4                      ;\
        ldx     [%sp + V9BIAS64 + 104], %i5                     ;\
        ldx     [%sp + V9BIAS64 + 112], %i6                     ;\
        ldx     [%sp + V9BIAS64 + 120], %i7                     ;\
        restored                                                ;\
        retry                                                   ;\
        SKIP(31-18-TT_TRACE_L_INS)                              ;\
        ba,a,pt %xcc, fault_64bit_##tail                        ;\
        .empty

/*
 * FILL_64bit_asi fills a 64-bit-wide register window from a 64-bit
 * wide address space via the designated asi.  It is used to fill
 * non-kernel windows.  The stack pointer is required to be eight-byte
 * aligned.
 */
#define FILL_64bit_asi(asi_num, tail)                           \
        mov     V9BIAS64 + 0, %g1                               ;\
2:      TT_TRACE_L(trace_win)                                   ;\
        ldxa    [%sp + %g1]asi_num, %l0                         ;\
        mov     V9BIAS64 + 8, %g2                               ;\
        ldxa    [%sp + %g2]asi_num, %l1                         ;\
        mov     V9BIAS64 + 16, %g3                              ;\
        ldxa    [%sp + %g3]asi_num, %l2                         ;\
        mov     V9BIAS64 + 24, %g4                              ;\
        ldxa    [%sp + %g4]asi_num, %l3                         ;\
        add     %sp, 32, %g5                                    ;\
        ldxa    [%g5 + %g1]asi_num, %l4                         ;\
        ldxa    [%g5 + %g2]asi_num, %l5                         ;\
        ldxa    [%g5 + %g3]asi_num, %l6                         ;\
        ldxa    [%g5 + %g4]asi_num, %l7                         ;\
        add     %g5, 32, %g5                                    ;\
        ldxa    [%g5 + %g1]asi_num, %i0                         ;\
        ldxa    [%g5 + %g2]asi_num, %i1                         ;\
        ldxa    [%g5 + %g3]asi_num, %i2                         ;\
        ldxa    [%g5 + %g4]asi_num, %i3                         ;\
        add     %g5, 32, %g5                                    ;\
        ldxa    [%g5 + %g1]asi_num, %i4                         ;\
        ldxa    [%g5 + %g2]asi_num, %i5                         ;\
        ldxa    [%g5 + %g3]asi_num, %i6                         ;\
        ldxa    [%g5 + %g4]asi_num, %i7                         ;\
        restored                                                ;\
        retry                                                   ;\
        SKIP(31-25-TT_TRACE_L_INS)                              ;\
        ba,a,pt %xcc, fault_64bit_##tail                        ;\
        .empty


/*
 * SPILL_mixed spills either size window, depending on
 * whether %sp is even or odd, to a 32-bit address space.
 * This may only be used in conjunction with SPILL_32bit/
 * FILL_64bit.
 * Clear upper 32 bits of %sp if it is odd.
 * We won't need to clear them in 64 bit kernel.
 */
#define SPILL_mixed                                             \
        btst    1, %sp                                          ;\
        bz,a,pt %xcc, 1b                                        ;\
        srl     %sp, 0, %sp                                     ;\
        ba,pt   %xcc, 2b                                        ;\
        nop                                                     ;\
        .align  128

/*
 * FILL_mixed(ASI) fills either size window, depending on
 * whether %sp is even or odd, from a 32-bit address space.
 * This may only be used in conjunction with FILL_32bit/
 * FILL_64bit. New versions of FILL_mixed_{tt1,asi} would be
 * needed for use with FILL_{32,64}bit_{tt1,asi}. Particular
 * attention should be paid to the instructions that belong
 * in the delay slots of the branches depending on the type
 * of fill handler being branched to.
 * Clear upper 32 bits of %sp if it is odd.
 * We won't need to clear them in 64 bit kernel.
 */
#define FILL_mixed                                              \
        btst    1, %sp                                          ;\
        bz,a,pt %xcc, 1b                                        ;\
        srl     %sp, 0, %sp                                     ;\
        ba,pt   %xcc, 2b                                        ;\
        nop                                                     ;\
        .align  128


/*
 * SPILL_32clean/SPILL_64clean spill 32-bit and 64-bit register windows,
 * respectively, into the address space via the designated asi.  The
 * unbiased stack pointer is required to be eight-byte aligned (even for
 * the 32-bit case even though this code does not require such strict
 * alignment).
 *
 * With SPARC v9 the spill trap takes precedence over the cleanwin trap
 * so when cansave == 0, canrestore == 6, and cleanwin == 6 the next save
 * will cause cwp + 2 to be spilled but will not clean cwp + 1.  That
 * window may contain kernel data so in user_rtt we set wstate to call
 * these spill handlers on the first user spill trap.  These handler then
 * spill the appropriate window but also back up a window and clean the
 * window that didn't get a cleanwin trap.
 */
#define SPILL_32clean(asi_num, tail)                            \
        srl     %sp, 0, %sp                                     ;\
        sta     %l0, [%sp + %g0]asi_num                         ;\
        mov     4, %g1                                          ;\
        sta     %l1, [%sp + %g1]asi_num                         ;\
        mov     8, %g2                                          ;\
        sta     %l2, [%sp + %g2]asi_num                         ;\
        mov     12, %g3                                         ;\
        sta     %l3, [%sp + %g3]asi_num                         ;\
        add     %sp, 16, %g4                                    ;\
        sta     %l4, [%g4 + %g0]asi_num                         ;\
        sta     %l5, [%g4 + %g1]asi_num                         ;\
        sta     %l6, [%g4 + %g2]asi_num                         ;\
        sta     %l7, [%g4 + %g3]asi_num                         ;\
        add     %g4, 16, %g4                                    ;\
        sta     %i0, [%g4 + %g0]asi_num                         ;\
        sta     %i1, [%g4 + %g1]asi_num                         ;\
        sta     %i2, [%g4 + %g2]asi_num                         ;\
        sta     %i3, [%g4 + %g3]asi_num                         ;\
        add     %g4, 16, %g4                                    ;\
        sta     %i4, [%g4 + %g0]asi_num                         ;\
        sta     %i5, [%g4 + %g1]asi_num                         ;\
        sta     %i6, [%g4 + %g2]asi_num                         ;\
        sta     %i7, [%g4 + %g3]asi_num                         ;\
        TT_TRACE_L(trace_win)                                   ;\
        b       .spill_clean                                    ;\
          mov   WSTATE_USER32, %g7                              ;\
        SKIP(31-25-TT_TRACE_L_INS)                              ;\
        ba,a,pt %xcc, fault_32bit_##tail                        ;\
        .empty

#define SPILL_64clean(asi_num, tail)                            \
        mov     0 + V9BIAS64, %g1                               ;\
        stxa    %l0, [%sp + %g1]asi_num                         ;\
        mov     8 + V9BIAS64, %g2                               ;\
        stxa    %l1, [%sp + %g2]asi_num                         ;\
        mov     16 + V9BIAS64, %g3                              ;\
        stxa    %l2, [%sp + %g3]asi_num                         ;\
        mov     24 + V9BIAS64, %g4                              ;\
        stxa    %l3, [%sp + %g4]asi_num                         ;\
        add     %sp, 32, %g5                                    ;\
        stxa    %l4, [%g5 + %g1]asi_num                         ;\
        stxa    %l5, [%g5 + %g2]asi_num                         ;\
        stxa    %l6, [%g5 + %g3]asi_num                         ;\
        stxa    %l7, [%g5 + %g4]asi_num                         ;\
        add     %g5, 32, %g5                                    ;\
        stxa    %i0, [%g5 + %g1]asi_num                         ;\
        stxa    %i1, [%g5 + %g2]asi_num                         ;\
        stxa    %i2, [%g5 + %g3]asi_num                         ;\
        stxa    %i3, [%g5 + %g4]asi_num                         ;\
        add     %g5, 32, %g5                                    ;\
        stxa    %i4, [%g5 + %g1]asi_num                         ;\
        stxa    %i5, [%g5 + %g2]asi_num                         ;\
        stxa    %i6, [%g5 + %g3]asi_num                         ;\
        stxa    %i7, [%g5 + %g4]asi_num                         ;\
        TT_TRACE_L(trace_win)                                   ;\
        b       .spill_clean                                    ;\
          mov   WSTATE_USER64, %g7                              ;\
        SKIP(31-25-TT_TRACE_L_INS)                              ;\
        ba,a,pt %xcc, fault_64bit_##tail                        ;\
        .empty


/*
 * Floating point disabled.
 */
#define FP_DISABLED_TRAP                \
        TT_TRACE(trace_gen)             ;\
        ba,pt   %xcc,.fp_disabled       ;\
        nop                             ;\
        .align  32

/*
 * Floating point exceptions.
 */
#define FP_IEEE_TRAP                    \
        TT_TRACE(trace_gen)             ;\
        ba,pt   %xcc,.fp_ieee_exception ;\
        nop                             ;\
        .align  32

#define FP_TRAP                         \
        TT_TRACE(trace_gen)             ;\
        ba,pt   %xcc,.fp_exception      ;\
        nop                             ;\
        .align  32

/*
 * ECACHE_ECC error traps at level 0 and level 1
 */
#define ECACHE_ECC(table_name)          \
        .global table_name              ;\
table_name:                             ;\
        membar  #Sync                   ;\
        set     trap, %g1               ;\
        rdpr    %tt, %g3                ;\
        ba,pt   %xcc, sys_trap          ;\
        sub     %g0, 1, %g4             ;\
        .align  32

/*
 * illegal instruction trap
 */
#define ILLTRAP_INSTR                     \
        membar  #Sync                     ;\
        TT_TRACE(trace_gen)               ;\
        or      %g0, P_UTRAP4, %g2        ;\
        or      %g0, T_UNIMP_INSTR, %g3   ;\
        sethi   %hi(.check_v9utrap), %g4  ;\
        jmp     %g4 + %lo(.check_v9utrap) ;\
        nop                               ;\
        .align  32

/*
 * tag overflow trap
 */
#define TAG_OVERFLOW                      \
        TT_TRACE(trace_gen)               ;\
        or      %g0, P_UTRAP10, %g2       ;\
        or      %g0, T_TAG_OVERFLOW, %g3  ;\
        sethi   %hi(.check_v9utrap), %g4  ;\
        jmp     %g4 + %lo(.check_v9utrap) ;\
        nop                               ;\
        .align  32

/*
 * divide by zero trap
 */
#define DIV_BY_ZERO                       \
        TT_TRACE(trace_gen)               ;\
        or      %g0, P_UTRAP11, %g2       ;\
        or      %g0, T_IDIV0, %g3         ;\
        sethi   %hi(.check_v9utrap), %g4  ;\
        jmp     %g4 + %lo(.check_v9utrap) ;\
        nop                               ;\
        .align  32

/*
 * trap instruction for V9 user trap handlers
 */
#define TRAP_INSTR                        \
        TT_TRACE(trace_gen)               ;\
        or      %g0, T_SOFTWARE_TRAP, %g3 ;\
        sethi   %hi(.check_v9utrap), %g4  ;\
        jmp     %g4 + %lo(.check_v9utrap) ;\
        nop                               ;\
        .align  32
#define TRP4    TRAP_INSTR; TRAP_INSTR; TRAP_INSTR; TRAP_INSTR

/*
 * LEVEL_INTERRUPT is for level N interrupts.
 * VECTOR_INTERRUPT is for the vector trap.
 */
#define LEVEL_INTERRUPT(level)          \
        .global tt_pil##level           ;\
tt_pil##level:                  ;\
        ba,pt   %xcc, pil_interrupt     ;\
        mov     level, %g4              ;\
        .align  32

#define LEVEL14_INTERRUPT                       \
        ba      pil14_interrupt                 ;\
        mov     PIL_14, %g4                     ;\
        .align  32

#define        LEVEL15_INTERRUPT                       \
       ba      pil15_interrupt                 ;\
       mov     PIL_15, %g4                     ;\
       .align  32

#define CPU_MONDO                       \
        ba,a,pt %xcc, cpu_mondo         ;\
        .align  32

#define DEV_MONDO                       \
        ba,a,pt %xcc, dev_mondo         ;\
        .align  32

/*
 * We take over the rtba after we set our trap table and
 * fault status area. The watchdog reset trap is now handled by the OS.
 */
#define WATCHDOG_RESET                  \
        mov     PTL1_BAD_WATCHDOG, %g1  ;\
        ba,a,pt %xcc, .watchdog_trap    ;\
        .align  32

/*
 * RED is for traps that use the red mode handler.
 * We should never see these either.
 */
#define RED                     \
        mov     PTL1_BAD_RED, %g1       ;\
        ba,a,pt %xcc, .watchdog_trap    ;\
        .align  32


/*
 * MMU Trap Handlers.
 */

/*
 * synthesize for trap(): SFSR in %g3
 */
#define IMMU_EXCEPTION                                                  \
        MMU_FAULT_STATUS_AREA(%g3)                                      ;\
        rdpr    %tpc, %g2                                               ;\
        ldx     [%g3 + MMFSA_I_TYPE], %g1                               ;\
        ldx     [%g3 + MMFSA_I_CTX], %g3                                ;\
        sllx    %g3, SFSR_CTX_SHIFT, %g3                                ;\
        or      %g3, %g1, %g3                                           ;\
        ba,pt   %xcc, .mmu_exception_end                                ;\
        mov     T_INSTR_EXCEPTION, %g1                                  ;\
        .align  32

/*
 * synthesize for trap(): TAG_ACCESS in %g2, SFSR in %g3
 */
#define DMMU_EXCEPTION                                                  \
        ba,a,pt %xcc, .dmmu_exception                                   ;\
        .align  32

/*
 * synthesize for trap(): SFAR in %g2, SFSR in %g3
 */
#define DMMU_EXC_AG_PRIV                                                \
        MMU_FAULT_STATUS_AREA(%g3)                                      ;\
        ldx     [%g3 + MMFSA_D_ADDR], %g2                               ;\
        /* Fault type not available in MMU fault status area */         ;\
        mov     MMFSA_F_PRVACT, %g1                                     ;\
        ldx     [%g3 + MMFSA_D_CTX], %g3                                ;\
        sllx    %g3, SFSR_CTX_SHIFT, %g3                                ;\
        ba,pt   %xcc, .mmu_priv_exception                               ;\
        or      %g3, %g1, %g3                                           ;\
        .align  32

/*
 * synthesize for trap(): SFAR in %g2, SFSR in %g3
 */
#define DMMU_EXC_AG_NOT_ALIGNED                                         \
        MMU_FAULT_STATUS_AREA(%g3)                                      ;\
        ldx     [%g3 + MMFSA_D_ADDR], %g2                               ;\
        /* Fault type not available in MMU fault status area */         ;\
        mov     MMFSA_F_UNALIGN, %g1                                    ;\
        ldx     [%g3 + MMFSA_D_CTX], %g3                                ;\
        sllx    %g3, SFSR_CTX_SHIFT, %g3                                ;\
        ba,pt   %xcc, .mmu_exception_not_aligned                        ;\
        or      %g3, %g1, %g3                   /* SFSR */              ;\
        .align  32
/*
 * SPARC V9 IMPL. DEP. #109(1) and (2) and #110(1) and (2)
 */

/*
 * synthesize for trap(): SFAR in %g2, SFSR in %g3
 */
#define DMMU_EXC_LDDF_NOT_ALIGNED                                       \
        ba,a,pt %xcc, .dmmu_exc_lddf_not_aligned                        ;\
        .align  32
/*
 * synthesize for trap(): SFAR in %g2, SFSR in %g3
 */
#define DMMU_EXC_STDF_NOT_ALIGNED                                       \
        ba,a,pt %xcc, .dmmu_exc_stdf_not_aligned                        ;\
        .align  32

#if defined(cscope)
/*
 * Define labels to direct cscope quickly to labels that
 * are generated by macro expansion of DTLB_MISS().
 */
        .global tt0_dtlbmiss
tt0_dtlbmiss:
        .global tt1_dtlbmiss
tt1_dtlbmiss:
        nop
#endif

/*
 * Data miss handler (must be exactly 32 instructions)
 *
 * This handler is invoked only if the hypervisor has been instructed
 * not to do any TSB walk.
 *
 * Kernel and invalid context cases are handled by the sfmmu_kdtlb_miss
 * handler.
 *
 * User TLB miss handling depends upon whether a user process has one or
 * two TSBs. User TSB information (physical base and size code) is kept
 * in two dedicated scratchpad registers. Absence of a user TSB (primarily
 * second TSB) is indicated by a negative value (-1) in that register.
 */

/*
 * synthesize for miss handler: pseudo-tag access in %g2 (with context "type"
 * (0=kernel, 1=invalid, or 2=user) rather than context ID)
 */
#define DTLB_MISS(table_name)                                           ;\
        .global table_name##_dtlbmiss                                   ;\
table_name##_dtlbmiss:                                          ;\
        GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3)     /* 8 instr */           ;\
        cmp     %g3, INVALID_CONTEXT                                    ;\
        ble,pn  %xcc, sfmmu_kdtlb_miss                                  ;\
          srlx  %g2, TAG_VALO_SHIFT, %g7        /* g7 = tsb tag */      ;\
        mov     SCRATCHPAD_UTSBREG2, %g1                                ;\
        ldxa    [%g1]ASI_SCRATCHPAD, %g1        /* get 2nd tsbreg */    ;\
        brgez,pn %g1, sfmmu_udtlb_slowpath      /* branch if 2 TSBs */  ;\
          nop                                                           ;\
        GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)    /* 11 instr */          ;\
        ba,pt   %xcc, sfmmu_udtlb_fastpath      /* no 4M TSB, miss */   ;\
          srlx  %g2, TAG_VALO_SHIFT, %g7        /* g7 = tsb tag */      ;\
        .align 128


#if defined(cscope)
/*
 * Define labels to direct cscope quickly to labels that
 * are generated by macro expansion of ITLB_MISS().
 */
        .global tt0_itlbmiss
tt0_itlbmiss:
        .global tt1_itlbmiss
tt1_itlbmiss:
        nop
#endif

/*
 * Instruction miss handler.
 *
 * This handler is invoked only if the hypervisor has been instructed
 * not to do any TSB walk.
 *
 * ldda instructions will have their ASI patched
 * by sfmmu_patch_ktsb at runtime.
 * MUST be EXACTLY 32 instructions or we'll break.
 */

/*
 * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type"
 * (0=kernel, 1=invalid, or 2=user) rather than context ID)
 */
#define ITLB_MISS(table_name)                                            \
        .global table_name##_itlbmiss                                   ;\
table_name##_itlbmiss:                                          ;\
        GET_MMU_I_PTAGACC_CTXTYPE(%g2, %g3)     /* 8 instr */           ;\
        cmp     %g3, INVALID_CONTEXT                                    ;\
        ble,pn  %xcc, sfmmu_kitlb_miss                                  ;\
          srlx  %g2, TAG_VALO_SHIFT, %g7        /* g7 = tsb tag */      ;\
        mov     SCRATCHPAD_UTSBREG2, %g1                                ;\
        ldxa    [%g1]ASI_SCRATCHPAD, %g1        /* get 2nd tsbreg */    ;\
        brgez,pn %g1, sfmmu_uitlb_slowpath      /* branch if 2 TSBs */  ;\
          nop                                                           ;\
        GET_1ST_TSBE_PTR(%g2, %g1, %g4, %g5)    /* 11 instr */          ;\
        ba,pt   %xcc, sfmmu_uitlb_fastpath      /* no 4M TSB, miss */   ;\
          srlx  %g2, TAG_VALO_SHIFT, %g7        /* g7 = tsb tag */      ;\
        .align 128

#define DTSB_MISS \
        GOTO_TT(sfmmu_slow_dmmu_miss,trace_dmmu)

#define ITSB_MISS \
        GOTO_TT(sfmmu_slow_immu_miss,trace_immu)

/*
 * This macro is the first level handler for fast protection faults.
 * It first demaps the tlb entry which generated the fault and then
 * attempts to set the modify bit on the hash.  It needs to be
 * exactly 32 instructions.
 */
/*
 * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type"
 * (0=kernel, 1=invalid, or 2=user) rather than context ID)
 */
#define DTLB_PROT                                                        \
        GET_MMU_D_PTAGACC_CTXTYPE(%g2, %g3)     /* 8 instr */           ;\
        /*                                                              ;\
         *   g2 = pseudo-tag access register (ctx type rather than ctx ID) ;\
         *   g3 = ctx type (0, 1, or 2)                                 ;\
         */                                                             ;\
        TT_TRACE(trace_dataprot)        /* 2 instr ifdef TRAPTRACE */   ;\
                                        /* clobbers g1 and g6 XXXQ? */  ;\
        brnz,pt %g3, sfmmu_uprot_trap           /* user trap */         ;\
          nop                                                           ;\
        ba,a,pt %xcc, sfmmu_kprot_trap          /* kernel trap */       ;\
        .align 128

#define DMMU_EXCEPTION_TL1                                              ;\
        ba,a,pt %xcc, mmu_trap_tl1                                      ;\
        .align 32

#define MISALIGN_ADDR_TL1                                               ;\
        ba,a,pt %xcc, mmu_trap_tl1                                      ;\
        .align 32

/*
 * Trace a tsb hit
 * g1 = tsbe pointer (in/clobbered)
 * g2 = tag access register (in)
 * g3 - g4 = scratch (clobbered)
 * g5 = tsbe data (in)
 * g6 = scratch (clobbered)
 * g7 = pc we jumped here from (in)
 * ttextra = value to OR in to trap type (%tt) (in)
 */
#ifdef TRAPTRACE
#define TRACE_TSBHIT(ttextra)                                            \
        membar  #Sync                                                   ;\
        sethi   %hi(FLUSH_ADDR), %g6                                    ;\
        flush   %g6                                                     ;\
        TRACE_PTR(%g3, %g6)                                             ;\
        GET_TRACE_TICK(%g6, %g4)                                        ;\
        stxa    %g6, [%g3 + TRAP_ENT_TICK]%asi                          ;\
        stna    %g2, [%g3 + TRAP_ENT_SP]%asi    /* tag access */        ;\
        stna    %g5, [%g3 + TRAP_ENT_F1]%asi    /* tsb data */          ;\
        rdpr    %tnpc, %g6                                              ;\
        stna    %g6, [%g3 + TRAP_ENT_F2]%asi                            ;\
        stna    %g1, [%g3 + TRAP_ENT_F3]%asi    /* tsb pointer */       ;\
        stna    %g0, [%g3 + TRAP_ENT_F4]%asi                            ;\
        rdpr    %tpc, %g6                                               ;\
        stna    %g6, [%g3 + TRAP_ENT_TPC]%asi                           ;\
        TRACE_SAVE_TL_GL_REGS(%g3, %g6)                                 ;\
        rdpr    %tt, %g6                                                ;\
        or      %g6, (ttextra), %g1                                     ;\
        stha    %g1, [%g3 + TRAP_ENT_TT]%asi                            ;\
        MMU_FAULT_STATUS_AREA(%g4)                                      ;\
        mov     MMFSA_D_ADDR, %g1                                       ;\
        cmp     %g6, FAST_IMMU_MISS_TT                                  ;\
        move    %xcc, MMFSA_I_ADDR, %g1                                 ;\
        cmp     %g6, T_INSTR_MMU_MISS                                   ;\
        move    %xcc, MMFSA_I_ADDR, %g1                                 ;\
        ldx     [%g4 + %g1], %g1                                        ;\
        stxa    %g1, [%g3 + TRAP_ENT_TSTATE]%asi /* fault addr */       ;\
        mov     MMFSA_D_CTX, %g1                                        ;\
        cmp     %g6, FAST_IMMU_MISS_TT                                  ;\
        move    %xcc, MMFSA_I_CTX, %g1                                  ;\
        cmp     %g6, T_INSTR_MMU_MISS                                   ;\
        move    %xcc, MMFSA_I_CTX, %g1                                  ;\
        ldx     [%g4 + %g1], %g1                                        ;\
        stna    %g1, [%g3 + TRAP_ENT_TR]%asi                            ;\
        TRACE_NEXT(%g3, %g4, %g6)
#else
#define TRACE_TSBHIT(ttextra)
#endif


/*
 * =======================================================================
 *              SPARC V9 TRAP TABLE
 *
 * The trap table is divided into two halves: the first half is used when
 * taking traps when TL=0; the second half is used when taking traps from
 * TL>0. Note that handlers in the second half of the table might not be able
 * to make the same assumptions as handlers in the first half of the table.
 *
 * Worst case trap nesting so far:
 *
 *      at TL=0 client issues software trap requesting service
 *      at TL=1 nucleus wants a register window
 *      at TL=2 register window clean/spill/fill takes a TLB miss
 *      at TL=3 processing TLB miss
 *      at TL=4 handle asynchronous error
 *
 * Note that a trap from TL=4 to TL=5 places Spitfire in "RED mode".
 *
 * =======================================================================
 */
        .section ".text"
        .align  4
        .global trap_table, scb, trap_table0, trap_table1, etrap_table
        .type   trap_table, #object
        .type   trap_table0, #object
        .type   trap_table1, #object
        .type   scb, #object
trap_table:
scb:
trap_table0:
        /* hardware traps */
        NOT;                            /* 000  reserved */
        RED;                            /* 001  power on reset */
        WATCHDOG_RESET;                 /* 002  watchdog reset */
        RED;                            /* 003  externally initiated reset */
        RED;                            /* 004  software initiated reset */
        RED;                            /* 005  red mode exception */
        NOT; NOT;                       /* 006 - 007 reserved */
        IMMU_EXCEPTION;                 /* 008  instruction access exception */
        ITSB_MISS;                      /* 009  instruction access MMU miss */
        NOT;                            /* 00A  reserved */
        NOT; NOT4;                      /* 00B - 00F reserved */
        ILLTRAP_INSTR;                  /* 010  illegal instruction */
        TRAP(T_PRIV_INSTR);             /* 011  privileged opcode */
        TRAP(T_UNIMP_LDD);              /* 012  unimplemented LDD */
        TRAP(T_UNIMP_STD);              /* 013  unimplemented STD */
        NOT4; NOT4; NOT4;               /* 014 - 01F reserved */
        FP_DISABLED_TRAP;               /* 020  fp disabled */
        FP_IEEE_TRAP;                   /* 021  fp exception ieee 754 */
        FP_TRAP;                        /* 022  fp exception other */
        TAG_OVERFLOW;                   /* 023  tag overflow */
        CLEAN_WINDOW;                   /* 024 - 027 clean window */
        DIV_BY_ZERO;                    /* 028  division by zero */
        NOT;                            /* 029  internal processor error */
        NOT; NOT; NOT4;                 /* 02A - 02F reserved */
        DMMU_EXCEPTION;                 /* 030  data access exception */
        DTSB_MISS;                      /* 031  data access MMU miss */
        NOT;                            /* 032  reserved */
        NOT;                            /* 033  data access protection */
        DMMU_EXC_AG_NOT_ALIGNED;        /* 034  mem address not aligned */
        DMMU_EXC_LDDF_NOT_ALIGNED;      /* 035  LDDF mem address not aligned */
        DMMU_EXC_STDF_NOT_ALIGNED;      /* 036  STDF mem address not aligned */
        DMMU_EXC_AG_PRIV;               /* 037  privileged action */
        NOT;                            /* 038  LDQF mem address not aligned */
        NOT;                            /* 039  STQF mem address not aligned */
        NOT; NOT; NOT4;                 /* 03A - 03F reserved */
        NOT;                            /* 040  async data error */
        LEVEL_INTERRUPT(1);             /* 041  interrupt level 1 */
        LEVEL_INTERRUPT(2);             /* 042  interrupt level 2 */
        LEVEL_INTERRUPT(3);             /* 043  interrupt level 3 */
        LEVEL_INTERRUPT(4);             /* 044  interrupt level 4 */
        LEVEL_INTERRUPT(5);             /* 045  interrupt level 5 */
        LEVEL_INTERRUPT(6);             /* 046  interrupt level 6 */
        LEVEL_INTERRUPT(7);             /* 047  interrupt level 7 */
        LEVEL_INTERRUPT(8);             /* 048  interrupt level 8 */
        LEVEL_INTERRUPT(9);             /* 049  interrupt level 9 */
        LEVEL_INTERRUPT(10);            /* 04A  interrupt level 10 */
        LEVEL_INTERRUPT(11);            /* 04B  interrupt level 11 */
        LEVEL_INTERRUPT(12);            /* 04C  interrupt level 12 */
        LEVEL_INTERRUPT(13);            /* 04D  interrupt level 13 */
        LEVEL14_INTERRUPT;              /* 04E  interrupt level 14 */
        LEVEL15_INTERRUPT;              /* 04F  interrupt level 15 */
        NOT4; NOT4; NOT4; NOT4;         /* 050 - 05F reserved */
        NOT;                            /* 060  interrupt vector */
        GOTO(kmdb_trap);                /* 061  PA watchpoint */
        GOTO(kmdb_trap);                /* 062  VA watchpoint */
        NOT;                            /* 063  reserved */
        ITLB_MISS(tt0);                 /* 064  instruction access MMU miss */
        DTLB_MISS(tt0);                 /* 068  data access MMU miss */
        DTLB_PROT;                      /* 06C  data access protection */
        NOT;                            /* 070  reserved */
        NOT;                            /* 071  reserved */
        NOT;                            /* 072  reserved */
        NOT;                            /* 073  reserved */
        NOT4; NOT4                      /* 074 - 07B reserved */
        CPU_MONDO;                      /* 07C  cpu_mondo */
        DEV_MONDO;                      /* 07D  dev_mondo */
        GOTO_TT(resumable_error, trace_gen);    /* 07E  resumable error */
        GOTO_TT(nonresumable_error, trace_gen); /* 07F  non-reasumable error */
        NOT4;                           /* 080  spill 0 normal */
        SPILL_32bit_asi(ASI_AIUP,sn0);  /* 084  spill 1 normal */
        SPILL_64bit_asi(ASI_AIUP,sn0);  /* 088  spill 2 normal */
        SPILL_32clean(ASI_AIUP,sn0);    /* 08C  spill 3 normal */
        SPILL_64clean(ASI_AIUP,sn0);    /* 090  spill 4 normal */
        SPILL_32bit(not);               /* 094  spill 5 normal */
        SPILL_64bit(not);               /* 098  spill 6 normal */
        SPILL_mixed;                    /* 09C  spill 7 normal */
        NOT4;                           /* 0A0  spill 0 other */
        SPILL_32bit_asi(ASI_AIUS,so0);  /* 0A4  spill 1 other */
        SPILL_64bit_asi(ASI_AIUS,so0);  /* 0A8  spill 2 other */
        SPILL_32bit_asi(ASI_AIUS,so0);  /* 0AC  spill 3 other */
        SPILL_64bit_asi(ASI_AIUS,so0);  /* 0B0  spill 4 other */
        NOT4;                           /* 0B4  spill 5 other */
        NOT4;                           /* 0B8  spill 6 other */
        NOT4;                           /* 0BC  spill 7 other */
        NOT4;                           /* 0C0  fill 0 normal */
        FILL_32bit_asi(ASI_AIUP,fn0);   /* 0C4  fill 1 normal */
        FILL_64bit_asi(ASI_AIUP,fn0);   /* 0C8  fill 2 normal */
        FILL_32bit_asi(ASI_AIUP,fn0);   /* 0CC  fill 3 normal */
        FILL_64bit_asi(ASI_AIUP,fn0);   /* 0D0  fill 4 normal */
        FILL_32bit(not);                /* 0D4  fill 5 normal */
        FILL_64bit(not);                /* 0D8  fill 6 normal */
        FILL_mixed;                     /* 0DC  fill 7 normal */
        NOT4;                           /* 0E0  fill 0 other */
        NOT4;                           /* 0E4  fill 1 other */
        NOT4;                           /* 0E8  fill 2 other */
        NOT4;                           /* 0EC  fill 3 other */
        NOT4;                           /* 0F0  fill 4 other */
        NOT4;                           /* 0F4  fill 5 other */
        NOT4;                           /* 0F8  fill 6 other */
        NOT4;                           /* 0FC  fill 7 other */
        /* user traps */
        GOTO(syscall_trap_4x);          /* 100  old system call */
        TRAP(T_BREAKPOINT);             /* 101  user breakpoint */
        TRAP(T_DIV0);                   /* 102  user divide by zero */
        GOTO(.flushw);                  /* 103  flush windows */
        GOTO(.clean_windows);           /* 104  clean windows */
        BAD;                            /* 105  range check ?? */
        GOTO(.fix_alignment);           /* 106  do unaligned references */
        BAD;                            /* 107  unused */
        SYSCALL_TRAP32;                 /* 108  ILP32 system call on LP64 */
        GOTO(set_trap0_addr);           /* 109  set trap0 address */
        BAD; BAD; BAD4;                 /* 10A - 10F unused */
        TRP4; TRP4; TRP4; TRP4;         /* 110 - 11F V9 user trap handlers */
        GOTO(.getcc);                   /* 120  get condition codes */
        GOTO(.setcc);                   /* 121  set condition codes */
        GOTO(.getpsr);                  /* 122  get psr */
        GOTO(.setpsr);                  /* 123  set psr (some fields) */
        GOTO(get_timestamp);            /* 124  get timestamp */
        GOTO(get_virtime);              /* 125  get lwp virtual time */
        PRIV(self_xcall);               /* 126  self xcall */
        GOTO(get_hrestime);             /* 127  get hrestime */
        BAD;                            /* 128  ST_SETV9STACK */
        GOTO(.getlgrp);                 /* 129  get lgrpid */
        BAD; BAD; BAD4;                 /* 12A - 12F unused */
        BAD4; BAD4;                     /* 130 - 137 unused */
        DTRACE_PID;                     /* 138  dtrace pid tracing provider */
        BAD;                            /* 139  unused */
        DTRACE_RETURN;                  /* 13A  dtrace pid return probe */
        BAD; BAD4;                      /* 13B - 13F unused */
        SYSCALL_TRAP;                   /* 140  LP64 system call */
        SYSCALL(nosys);                 /* 141  unused system call trap */
#ifdef DEBUG_USER_TRAPTRACECTL
        GOTO(.traptrace_freeze);        /* 142  freeze traptrace */
        GOTO(.traptrace_unfreeze);      /* 143  unfreeze traptrace */
#else
        SYSCALL(nosys);                 /* 142  unused system call trap */
        SYSCALL(nosys);                 /* 143  unused system call trap */
#endif
        BAD4; BAD4; BAD4;               /* 144 - 14F unused */
        BAD4; BAD4; BAD4; BAD4;         /* 150 - 15F unused */
        BAD4; BAD4; BAD4; BAD4;         /* 160 - 16F unused */
        BAD;                            /* 170 - unused */
        BAD;                            /* 171 - unused */
        BAD; BAD;                       /* 172 - 173 unused */
        BAD4; BAD4;                     /* 174 - 17B unused */
#ifdef  PTL1_PANIC_DEBUG
        mov PTL1_BAD_DEBUG, %g1; GOTO(ptl1_panic);
                                        /* 17C  test ptl1_panic */
#else
        BAD;                            /* 17C  unused */
#endif  /* PTL1_PANIC_DEBUG */
        PRIV(kmdb_trap);                /* 17D  kmdb enter (L1-A) */
        PRIV(kmdb_trap);                /* 17E  kmdb breakpoint */
        PRIV(obp_bpt);                  /* 17F  obp breakpoint */
        /* reserved */
        NOT4; NOT4; NOT4; NOT4;         /* 180 - 18F reserved */
        NOT4; NOT4; NOT4; NOT4;         /* 190 - 19F reserved */
        NOT4; NOT4; NOT4; NOT4;         /* 1A0 - 1AF reserved */
        NOT4; NOT4; NOT4; NOT4;         /* 1B0 - 1BF reserved */
        NOT4; NOT4; NOT4; NOT4;         /* 1C0 - 1CF reserved */
        NOT4; NOT4; NOT4; NOT4;         /* 1D0 - 1DF reserved */
        NOT4; NOT4; NOT4; NOT4;         /* 1E0 - 1EF reserved */
        NOT4; NOT4; NOT4; NOT4;         /* 1F0 - 1FF reserved */
        .size   trap_table0, (.-trap_table0)
trap_table1:
        NOT4; NOT4;                     /* 000 - 007 unused */
        NOT;                            /* 008  instruction access exception */
        ITSB_MISS;                      /* 009  instruction access MMU miss */
        NOT;                            /* 00A  reserved */
        NOT; NOT4;                      /* 00B - 00F unused */
        NOT4; NOT4; NOT4; NOT4;         /* 010 - 01F unused */
        NOT4;                           /* 020 - 023 unused */
        CLEAN_WINDOW;                   /* 024 - 027 clean window */
        NOT4; NOT4;                     /* 028 - 02F unused */
        DMMU_EXCEPTION_TL1;             /* 030  data access exception */
        DTSB_MISS;                      /* 031  data access MMU miss */
        NOT;                            /* 032  reserved */
        NOT;                            /* 033  unused */
        MISALIGN_ADDR_TL1;              /* 034  mem address not aligned */
        NOT; NOT; NOT; NOT4; NOT4       /* 035 - 03F unused */
        NOT4; NOT4; NOT4; NOT4;         /* 040 - 04F unused */
        NOT4; NOT4; NOT4; NOT4;         /* 050 - 05F unused */
        NOT;                            /* 060  unused */
        GOTO(kmdb_trap_tl1);            /* 061  PA watchpoint */
        GOTO(kmdb_trap_tl1);            /* 062  VA watchpoint */
        NOT;                            /* 063  reserved */
        ITLB_MISS(tt1);                 /* 064  instruction access MMU miss */
        DTLB_MISS(tt1);                 /* 068  data access MMU miss */
        DTLB_PROT;                      /* 06C  data access protection */
        NOT;                            /* 070  reserved */
        NOT;                            /* 071  reserved */
        NOT;                            /* 072  reserved */
        NOT;                            /* 073  reserved */
        NOT4; NOT4;                     /* 074 - 07B reserved */
        NOT;                            /* 07C  reserved */
        NOT;                            /* 07D  reserved */
        NOT;                            /* 07E  resumable error */
        GOTO_TT(nonresumable_error, trace_gen); /* 07F  nonresumable error */
        NOTP4;                          /* 080  spill 0 normal */
        SPILL_32bit_tt1(ASI_AIUP,sn1);  /* 084  spill 1 normal */
        SPILL_64bit_tt1(ASI_AIUP,sn1);  /* 088  spill 2 normal */
        SPILL_32bit_tt1(ASI_AIUP,sn1);  /* 08C  spill 3 normal */
        SPILL_64bit_tt1(ASI_AIUP,sn1);  /* 090  spill 4 normal */
        NOTP4;                          /* 094  spill 5 normal */
        SPILL_64bit_ktt1(sk);           /* 098  spill 6 normal */
        SPILL_mixed_ktt1(sk);           /* 09C  spill 7 normal */
        NOTP4;                          /* 0A0  spill 0 other */
        SPILL_32bit_tt1(ASI_AIUS,so1);  /* 0A4  spill 1 other */
        SPILL_64bit_tt1(ASI_AIUS,so1);  /* 0A8  spill 2 other */
        SPILL_32bit_tt1(ASI_AIUS,so1);  /* 0AC  spill 3 other */
        SPILL_64bit_tt1(ASI_AIUS,so1);  /* 0B0  spill 4 other */
        NOTP4;                          /* 0B4  spill 5 other */
        NOTP4;                          /* 0B8  spill 6 other */
        NOTP4;                          /* 0BC  spill 7 other */
        NOT4;                           /* 0C0  fill 0 normal */
        NOT4;                           /* 0C4  fill 1 normal */
        NOT4;                           /* 0C8  fill 2 normal */
        NOT4;                           /* 0CC  fill 3 normal */
        NOT4;                           /* 0D0  fill 4 normal */
        NOT4;                           /* 0D4  fill 5 normal */
        NOT4;                           /* 0D8  fill 6 normal */
        NOT4;                           /* 0DC  fill 7 normal */
        NOT4; NOT4; NOT4; NOT4;         /* 0E0 - 0EF unused */
        NOT4; NOT4; NOT4; NOT4;         /* 0F0 - 0FF unused */
/*
 * Code running at TL>0 does not use soft traps, so
 * we can truncate the table here.
 * However:
 * sun4v uses (hypervisor) ta instructions at TL > 0, so
 * provide a safety net for now.
 */
        /* soft traps */
        BAD4; BAD4; BAD4; BAD4;         /* 100 - 10F unused */
        BAD4; BAD4; BAD4; BAD4;         /* 110 - 11F unused */
        BAD4; BAD4; BAD4; BAD4;         /* 120 - 12F unused */
        BAD4; BAD4; BAD4; BAD4;         /* 130 - 13F unused */
        BAD4; BAD4; BAD4; BAD4;         /* 140 - 14F unused */
        BAD4; BAD4; BAD4; BAD4;         /* 150 - 15F unused */
        BAD4; BAD4; BAD4; BAD4;         /* 160 - 16F unused */
        BAD4; BAD4; BAD4; BAD4;         /* 170 - 17F unused */
        /* reserved */
        NOT4; NOT4; NOT4; NOT4;         /* 180 - 18F reserved */
        NOT4; NOT4; NOT4; NOT4;         /* 190 - 19F reserved */
        NOT4; NOT4; NOT4; NOT4;         /* 1A0 - 1AF reserved */
        NOT4; NOT4; NOT4; NOT4;         /* 1B0 - 1BF reserved */
        NOT4; NOT4; NOT4; NOT4;         /* 1C0 - 1CF reserved */
        NOT4; NOT4; NOT4; NOT4;         /* 1D0 - 1DF reserved */
        NOT4; NOT4; NOT4; NOT4;         /* 1E0 - 1EF reserved */
        NOT4; NOT4; NOT4; NOT4;         /* 1F0 - 1FF reserved */
etrap_table:
        .size   trap_table1, (.-trap_table1)
        .size   trap_table, (.-trap_table)
        .size   scb, (.-scb)

/*
 * We get to exec_fault in the case of an instruction miss and tte
 * has no execute bit set.  We go to tl0 to handle it.
 *
 * g1 = tsbe pointer (in/clobbered)
 * g2 = tag access register (in)
 * g3 - g4 = scratch (clobbered)
 * g5 = tsbe data (in)
 * g6 = scratch (clobbered)
 * g7 = pc we jumped here from (in)
 */
/*
 * synthesize for miss handler: TAG_ACCESS in %g2 (with context "type"
 * (0=kernel, 1=invalid, or 2=user) rather than context ID)
 */
        ALTENTRY(exec_fault)
        TRACE_TSBHIT(TT_MMU_EXEC)
        MMU_FAULT_STATUS_AREA(%g4)
        ldx     [%g4 + MMFSA_I_ADDR], %g2       /* g2 = address */
        ldx     [%g4 + MMFSA_I_CTX], %g3        /* g3 = ctx */
        srlx    %g2, MMU_PAGESHIFT, %g2         ! align address to page boundry
        cmp     %g3, USER_CONTEXT_TYPE
        sllx    %g2, MMU_PAGESHIFT, %g2
        movgu   %icc, USER_CONTEXT_TYPE, %g3
        or      %g2, %g3, %g2                   /* TAG_ACCESS */
        mov     T_INSTR_MMU_MISS, %g3           ! arg2 = traptype
        set     trap, %g1
        ba,pt   %xcc, sys_trap
          mov   -1, %g4

.mmu_exception_not_aligned:
        /* %g2 = sfar, %g3 = sfsr */
        rdpr    %tstate, %g1
        btst    TSTATE_PRIV, %g1
        bnz,pn  %icc, 2f
        nop
        CPU_ADDR(%g1, %g4)                              ! load CPU struct addr
        ldn     [%g1 + CPU_THREAD], %g1                 ! load thread pointer
        ldn     [%g1 + T_PROCP], %g1                    ! load proc pointer
        ldn     [%g1 + P_UTRAPS], %g5                   ! are there utraps?
        brz,pt  %g5, 2f
        nop
        ldn     [%g5 + P_UTRAP15], %g5                  ! unaligned utrap?
        brz,pn  %g5, 2f
        nop
        btst    1, %sp
        bz,pt   %xcc, 1f                                ! 32 bit user program
        nop
        ba,pt   %xcc, .setup_v9utrap                    ! 64 bit user program
        nop
1:
        ba,pt   %xcc, .setup_utrap
        or      %g2, %g0, %g7
2:
        ba,pt   %xcc, .mmu_exception_end
        mov     T_ALIGNMENT, %g1

.mmu_priv_exception:
        rdpr    %tstate, %g1
        btst    TSTATE_PRIV, %g1
        bnz,pn  %icc, 1f
        nop
        CPU_ADDR(%g1, %g4)                              ! load CPU struct addr
        ldn     [%g1 + CPU_THREAD], %g1                 ! load thread pointer
        ldn     [%g1 + T_PROCP], %g1                    ! load proc pointer
        ldn     [%g1 + P_UTRAPS], %g5                   ! are there utraps?
        brz,pt  %g5, 1f
        nop
        ldn     [%g5 + P_UTRAP16], %g5
        brnz,pt %g5, .setup_v9utrap
        nop
1:
        mov     T_PRIV_INSTR, %g1

.mmu_exception_end:
        CPU_INDEX(%g4, %g5)
        set     cpu_core, %g5
        sllx    %g4, CPU_CORE_SHIFT, %g4
        add     %g4, %g5, %g4
        lduh    [%g4 + CPUC_DTRACE_FLAGS], %g5
        andcc   %g5, CPU_DTRACE_NOFAULT, %g0
        bz      1f
        or      %g5, CPU_DTRACE_BADADDR, %g5
        stuh    %g5, [%g4 + CPUC_DTRACE_FLAGS]
        done

1:
        sllx    %g3, 32, %g3
        or      %g3, %g1, %g3
        set     trap, %g1
        ba,pt   %xcc, sys_trap
        sub     %g0, 1, %g4

.fp_disabled:
        CPU_ADDR(%g1, %g4)                              ! load CPU struct addr
        ldn     [%g1 + CPU_THREAD], %g1                 ! load thread pointer
        rdpr    %tstate, %g4
        btst    TSTATE_PRIV, %g4
        bnz,a,pn %icc, ptl1_panic
          mov   PTL1_BAD_FPTRAP, %g1

        ldn     [%g1 + T_PROCP], %g1                    ! load proc pointer
        ldn     [%g1 + P_UTRAPS], %g5                   ! are there utraps?
        brz,a,pt %g5, 2f
          nop
        ldn     [%g5 + P_UTRAP7], %g5                   ! fp_disabled utrap?
        brz,a,pn %g5, 2f
          nop
        btst    1, %sp
        bz,a,pt %xcc, 1f                                ! 32 bit user program
          nop
        ba,a,pt %xcc, .setup_v9utrap                    ! 64 bit user program
          nop
1:
        ba,pt   %xcc, .setup_utrap
          or    %g0, %g0, %g7
2:
        set     fp_disabled, %g1
        ba,pt   %xcc, sys_trap
          sub   %g0, 1, %g4

.fp_ieee_exception:
        rdpr    %tstate, %g1
        btst    TSTATE_PRIV, %g1
        bnz,a,pn %icc, ptl1_panic
          mov   PTL1_BAD_FPTRAP, %g1
        CPU_ADDR(%g1, %g4)                              ! load CPU struct addr
        stx     %fsr, [%g1 + CPU_TMP1]
        ldx     [%g1 + CPU_TMP1], %g2
        ldn     [%g1 + CPU_THREAD], %g1                 ! load thread pointer
        ldn     [%g1 + T_PROCP], %g1                    ! load proc pointer
        ldn     [%g1 + P_UTRAPS], %g5                   ! are there utraps?
        brz,a,pt %g5, 1f
          nop
        ldn     [%g5 + P_UTRAP8], %g5
        brnz,a,pt %g5, .setup_v9utrap
          nop
1:
        set     _fp_ieee_exception, %g1
        ba,pt   %xcc, sys_trap
          sub   %g0, 1, %g4

/*
 * Register Inputs:
 *      %g5             user trap handler
 *      %g7             misaligned addr - for alignment traps only
 */
.setup_utrap:
        set     trap, %g1                       ! setup in case we go
        mov     T_FLUSH_PCB, %g3                ! through sys_trap on
        sub     %g0, 1, %g4                     ! the save instruction below

        /*
         * If the DTrace pid provider is single stepping a copied-out
         * instruction, t->t_dtrace_step will be set. In that case we need
         * to abort the single-stepping (since execution of the instruction
         * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
         */
        save    %sp, -SA(MINFRAME32), %sp       ! window for trap handler
        CPU_ADDR(%g1, %g4)                      ! load CPU struct addr
        ldn     [%g1 + CPU_THREAD], %g1         ! load thread pointer
        ldub    [%g1 + T_DTRACE_STEP], %g2      ! load t->t_dtrace_step
        rdpr    %tnpc, %l2                      ! arg1 == tnpc
        brz,pt  %g2, 1f
        rdpr    %tpc, %l1                       ! arg0 == tpc

        ldub    [%g1 + T_DTRACE_AST], %g2       ! load t->t_dtrace_ast
        ldn     [%g1 + T_DTRACE_NPC], %l2       ! arg1 = t->t_dtrace_npc (step)
        brz,pt  %g2, 1f
        st      %g0, [%g1 + T_DTRACE_FT]        ! zero all pid provider flags
        stub    %g2, [%g1 + T_ASTFLAG]          ! aston(t) if t->t_dtrace_ast
1:
        mov     %g7, %l3                        ! arg2 == misaligned address

        rdpr    %tstate, %g1                    ! cwp for trap handler
        rdpr    %cwp, %g4
        bclr    TSTATE_CWP_MASK, %g1
        wrpr    %g1, %g4, %tstate
        wrpr    %g0, %g5, %tnpc                 ! trap handler address
        FAST_TRAP_DONE
        /* NOTREACHED */

.check_v9utrap:
        rdpr    %tstate, %g1
        btst    TSTATE_PRIV, %g1
        bnz,a,pn %icc, 3f
          nop
        CPU_ADDR(%g4, %g1)                              ! load CPU struct addr
        ldn     [%g4 + CPU_THREAD], %g5                 ! load thread pointer
        ldn     [%g5 + T_PROCP], %g5                    ! load proc pointer
        ldn     [%g5 + P_UTRAPS], %g5                   ! are there utraps?

        cmp     %g3, T_SOFTWARE_TRAP
        bne,a,pt %icc, 1f
          nop

        brz,pt %g5, 3f                  ! if p_utraps == NULL goto trap()
          rdpr  %tt, %g3                ! delay - get actual hw trap type

        sub     %g3, 254, %g1           ! UT_TRAP_INSTRUCTION_16 = p_utraps[18]
        ba,pt   %icc, 2f
          smul  %g1, CPTRSIZE, %g2
1:
        brz,a,pt %g5, 3f                ! if p_utraps == NULL goto trap()
          nop

        cmp     %g3, T_UNIMP_INSTR
        bne,a,pt %icc, 2f
          nop

        mov     1, %g1
        st      %g1, [%g4 + CPU_TL1_HDLR] ! set CPU_TL1_HDLR
        rdpr    %tpc, %g1               ! ld trapping instruction using
        lduwa   [%g1]ASI_AIUP, %g1      ! "AS IF USER" ASI which could fault
        st      %g0, [%g4 + CPU_TL1_HDLR] ! clr CPU_TL1_HDLR

        sethi   %hi(0xc1c00000), %g4    ! setup mask for illtrap instruction
        andcc   %g1, %g4, %g4           ! and instruction with mask
        bnz,a,pt %icc, 3f               ! if %g4 == zero, %g1 is an ILLTRAP
          nop                           ! fall thru to setup
2:
        ldn     [%g5 + %g2], %g5
        brnz,a,pt %g5, .setup_v9utrap
          nop
3:
        set     trap, %g1
        ba,pt   %xcc, sys_trap
          sub   %g0, 1, %g4
        /* NOTREACHED */

/*
 * Register Inputs:
 *      %g5             user trap handler
 */
.setup_v9utrap:
        set     trap, %g1                       ! setup in case we go
        mov     T_FLUSH_PCB, %g3                ! through sys_trap on
        sub     %g0, 1, %g4                     ! the save instruction below

        /*
         * If the DTrace pid provider is single stepping a copied-out
         * instruction, t->t_dtrace_step will be set. In that case we need
         * to abort the single-stepping (since execution of the instruction
         * was interrupted) and use the value of t->t_dtrace_npc as the %npc.
         */
        save    %sp, -SA(MINFRAME64), %sp       ! window for trap handler
        CPU_ADDR(%g1, %g4)                      ! load CPU struct addr
        ldn     [%g1 + CPU_THREAD], %g1         ! load thread pointer
        ldub    [%g1 + T_DTRACE_STEP], %g2      ! load t->t_dtrace_step
        rdpr    %tnpc, %l7                      ! arg1 == tnpc
        brz,pt  %g2, 1f
        rdpr    %tpc, %l6                       ! arg0 == tpc

        ldub    [%g1 + T_DTRACE_AST], %g2       ! load t->t_dtrace_ast
        ldn     [%g1 + T_DTRACE_NPC], %l7       ! arg1 == t->t_dtrace_npc (step)
        brz,pt  %g2, 1f
        st      %g0, [%g1 + T_DTRACE_FT]        ! zero all pid provider flags
        stub    %g2, [%g1 + T_ASTFLAG]          ! aston(t) if t->t_dtrace_ast
1:
        rdpr    %tstate, %g2                    ! cwp for trap handler
        rdpr    %cwp, %g4
        bclr    TSTATE_CWP_MASK, %g2
        wrpr    %g2, %g4, %tstate

        ldn     [%g1 + T_PROCP], %g4            ! load proc pointer
        ldn     [%g4 + P_AS], %g4               ! load as pointer
        ldn     [%g4 + A_USERLIMIT], %g4        ! load as userlimit
        cmp     %l7, %g4                        ! check for single-step set
        bne,pt  %xcc, 4f
          nop
        ldn     [%g1 + T_LWP], %g1              ! load klwp pointer
        ld      [%g1 + PCB_STEP], %g4           ! load single-step flag
        cmp     %g4, STEP_ACTIVE                ! step flags set in pcb?
        bne,pt  %icc, 4f
          nop
        stn     %g5, [%g1 + PCB_TRACEPC]        ! save trap handler addr in pcb
        mov     %l7, %g4                        ! on entry to precise user trap
        add     %l6, 4, %l7                     ! handler, %l6 == pc, %l7 == npc
                                                ! at time of trap
        wrpr    %g0, %g4, %tnpc                 ! generate FLTBOUNDS,
                                                ! %g4 == userlimit
        FAST_TRAP_DONE
        /* NOTREACHED */
4:
        wrpr    %g0, %g5, %tnpc                 ! trap handler address
        FAST_TRAP_DONE_CHK_INTR
        /* NOTREACHED */

.fp_exception:
        CPU_ADDR(%g1, %g4)
        stx     %fsr, [%g1 + CPU_TMP1]
        ldx     [%g1 + CPU_TMP1], %g2

        /*
         * Cheetah takes unfinished_FPop trap for certain range of operands
         * to the "fitos" instruction. Instead of going through the slow
         * software emulation path, we try to simulate the "fitos" instruction
         * via "fitod" and "fdtos" provided the following conditions are met:
         *
         *      fpu_exists is set (if DEBUG)
         *      not in privileged mode
         *      ftt is unfinished_FPop
         *      NXM IEEE trap is not enabled
         *      instruction at %tpc is "fitos"
         *
         *  Usage:
         *      %g1     per cpu address
         *      %g2     %fsr
         *      %g6     user instruction
         *
         * Note that we can take a memory access related trap while trying
         * to fetch the user instruction. Therefore, we set CPU_TL1_HDLR
         * flag to catch those traps and let the SFMMU code deal with page
         * fault and data access exception.
         */
#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
        sethi   %hi(fpu_exists), %g7
        ld      [%g7 + %lo(fpu_exists)], %g7
        brz,pn %g7, .fp_exception_cont
          nop
#endif
        rdpr    %tstate, %g7                    ! branch if in privileged mode
        btst    TSTATE_PRIV, %g7
        bnz,pn  %xcc, .fp_exception_cont
        srl     %g2, FSR_FTT_SHIFT, %g7         ! extract ftt from %fsr
        and     %g7, (FSR_FTT>>FSR_FTT_SHIFT), %g7
        cmp     %g7, FTT_UNFIN
        set     FSR_TEM_NX, %g5
        bne,pn  %xcc, .fp_exception_cont        ! branch if NOT unfinished_FPop
          andcc %g2, %g5, %g0
        bne,pn  %xcc, .fp_exception_cont        ! branch if FSR_TEM_NX enabled
          rdpr  %tpc, %g5                       ! get faulting PC

        or      %g0, 1, %g7
        st      %g7, [%g1 + CPU_TL1_HDLR]       ! set tl1_hdlr flag
        lda     [%g5]ASI_USER, %g6              ! get user's instruction
        st      %g0, [%g1 + CPU_TL1_HDLR]       ! clear tl1_hdlr flag

        set     FITOS_INSTR_MASK, %g7
        and     %g6, %g7, %g7
        set     FITOS_INSTR, %g5
        cmp     %g7, %g5
        bne,pn  %xcc, .fp_exception_cont        ! branch if not FITOS_INSTR
         nop

        /*
         * This is unfinished FPops trap for "fitos" instruction. We
         * need to simulate "fitos" via "fitod" and "fdtos" instruction
         * sequence.
         *
         * We need a temporary FP register to do the conversion. Since
         * both source and destination operands for the "fitos" instruction
         * have to be within %f0-%f31, we use an FP register from the upper
         * half to guarantee that it won't collide with the source or the
         * dest operand. However, we do have to save and restore its value.
         *
         * We use %d62 as a temporary FP register for the conversion and
         * branch to appropriate instruction within the conversion tables
         * based upon the rs2 and rd values.
         */

        std     %d62, [%g1 + CPU_TMP1]          ! save original value

        srl     %g6, FITOS_RS2_SHIFT, %g7
        and     %g7, FITOS_REG_MASK, %g7
        set     _fitos_fitod_table, %g4
        sllx    %g7, 2, %g7
        jmp     %g4 + %g7
          ba,pt %xcc, _fitos_fitod_done
        .empty

_fitos_fitod_table:
          fitod %f0, %d62
          fitod %f1, %d62
          fitod %f2, %d62
          fitod %f3, %d62
          fitod %f4, %d62
          fitod %f5, %d62
          fitod %f6, %d62
          fitod %f7, %d62
          fitod %f8, %d62
          fitod %f9, %d62
          fitod %f10, %d62
          fitod %f11, %d62
          fitod %f12, %d62
          fitod %f13, %d62
          fitod %f14, %d62
          fitod %f15, %d62
          fitod %f16, %d62
          fitod %f17, %d62
          fitod %f18, %d62
          fitod %f19, %d62
          fitod %f20, %d62
          fitod %f21, %d62
          fitod %f22, %d62
          fitod %f23, %d62
          fitod %f24, %d62
          fitod %f25, %d62
          fitod %f26, %d62
          fitod %f27, %d62
          fitod %f28, %d62
          fitod %f29, %d62
          fitod %f30, %d62
          fitod %f31, %d62
_fitos_fitod_done:

        /*
         * Now convert data back into single precision
         */
        srl     %g6, FITOS_RD_SHIFT, %g7
        and     %g7, FITOS_REG_MASK, %g7
        set     _fitos_fdtos_table, %g4
        sllx    %g7, 2, %g7
        jmp     %g4 + %g7
          ba,pt %xcc, _fitos_fdtos_done
        .empty

_fitos_fdtos_table:
          fdtos %d62, %f0
          fdtos %d62, %f1
          fdtos %d62, %f2
          fdtos %d62, %f3
          fdtos %d62, %f4
          fdtos %d62, %f5
          fdtos %d62, %f6
          fdtos %d62, %f7
          fdtos %d62, %f8
          fdtos %d62, %f9
          fdtos %d62, %f10
          fdtos %d62, %f11
          fdtos %d62, %f12
          fdtos %d62, %f13
          fdtos %d62, %f14
          fdtos %d62, %f15
          fdtos %d62, %f16
          fdtos %d62, %f17
          fdtos %d62, %f18
          fdtos %d62, %f19
          fdtos %d62, %f20
          fdtos %d62, %f21
          fdtos %d62, %f22
          fdtos %d62, %f23
          fdtos %d62, %f24
          fdtos %d62, %f25
          fdtos %d62, %f26
          fdtos %d62, %f27
          fdtos %d62, %f28
          fdtos %d62, %f29
          fdtos %d62, %f30
          fdtos %d62, %f31
_fitos_fdtos_done:

        ldd     [%g1 + CPU_TMP1], %d62          ! restore %d62

#if DEBUG
        /*
         * Update FPop_unfinished trap kstat
         */
        set     fpustat+FPUSTAT_UNFIN_KSTAT, %g7
        ldx     [%g7], %g5
1:
        add     %g5, 1, %g6

        casxa   [%g7] ASI_N, %g5, %g6
        cmp     %g5, %g6
        bne,a,pn %xcc, 1b
          or    %g0, %g6, %g5

        /*
         * Update fpu_sim_fitos kstat
         */
        set     fpuinfo+FPUINFO_FITOS_KSTAT, %g7
        ldx     [%g7], %g5
1:
        add     %g5, 1, %g6

        casxa   [%g7] ASI_N, %g5, %g6
        cmp     %g5, %g6
        bne,a,pn %xcc, 1b
          or    %g0, %g6, %g5
#endif /* DEBUG */

        FAST_TRAP_DONE

.fp_exception_cont:
        /*
         * Let _fp_exception deal with simulating FPop instruction.
         * Note that we need to pass %fsr in %g2 (already read above).
         */

        set     _fp_exception, %g1
        ba,pt   %xcc, sys_trap
        sub     %g0, 1, %g4


/*
 * Register windows
 */
.flushw:
.clean_windows:
        rdpr    %tnpc, %g1
        wrpr    %g1, %tpc
        add     %g1, 4, %g1
        wrpr    %g1, %tnpc
        set     trap, %g1
        mov     T_FLUSH_PCB, %g3
        ba,pt   %xcc, sys_trap
        sub     %g0, 1, %g4

/*
 * .spill_clean: clean the previous window, restore the wstate, and
 * "done".
 *
 * Entry: %g7 contains new wstate
 */
.spill_clean:
        sethi   %hi(nwin_minus_one), %g5
        ld      [%g5 + %lo(nwin_minus_one)], %g5 ! %g5 = nwin - 1
        rdpr    %cwp, %g6                       ! %g6 = %cwp
        deccc   %g6                             ! %g6--
        movneg  %xcc, %g5, %g6                  ! if (%g6<0) %g6 = nwin-1
        wrpr    %g6, %cwp
        TT_TRACE_L(trace_win)
        clr     %l0
        clr     %l1
        clr     %l2
        clr     %l3
        clr     %l4
        clr     %l5
        clr     %l6
        clr     %l7
        wrpr    %g0, %g7, %wstate
        saved
        retry                   ! restores correct %cwp

.fix_alignment:
        CPU_ADDR(%g1, %g2)              ! load CPU struct addr to %g1 using %g2
        ldn     [%g1 + CPU_THREAD], %g1 ! load thread pointer
        ldn     [%g1 + T_PROCP], %g1
        mov     1, %g2
        stb     %g2, [%g1 + P_FIXALIGNMENT]
        FAST_TRAP_DONE

#define STDF_REG(REG, ADDR, TMP)                \
        sll     REG, 3, REG                     ;\
mark1:  set     start1, TMP                     ;\
        jmp     REG + TMP                       ;\
          nop                                   ;\
start1: ba,pt   %xcc, done1                     ;\
          std   %f0, [ADDR + CPU_TMP1]          ;\
        ba,pt   %xcc, done1                     ;\
          std   %f32, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f2, [ADDR + CPU_TMP1]          ;\
        ba,pt   %xcc, done1                     ;\
          std   %f34, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f4, [ADDR + CPU_TMP1]          ;\
        ba,pt   %xcc, done1                     ;\
          std   %f36, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f6, [ADDR + CPU_TMP1]          ;\
        ba,pt   %xcc, done1                     ;\
          std   %f38, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f8, [ADDR + CPU_TMP1]          ;\
        ba,pt   %xcc, done1                     ;\
          std   %f40, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f10, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f42, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f12, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f44, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f14, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f46, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f16, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f48, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f18, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f50, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f20, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f52, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f22, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f54, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f24, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f56, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f26, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f58, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f28, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f60, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f30, [ADDR + CPU_TMP1]         ;\
        ba,pt   %xcc, done1                     ;\
          std   %f62, [ADDR + CPU_TMP1]         ;\
done1:

#define LDDF_REG(REG, ADDR, TMP)                \
        sll     REG, 3, REG                     ;\
mark2:  set     start2, TMP                     ;\
        jmp     REG + TMP                       ;\
          nop                                   ;\
start2: ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f0          ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f32         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f2          ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f34         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f4          ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f36         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f6          ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f38         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f8          ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f40         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f10         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f42         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f12         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f44         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f14         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f46         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f16         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f48         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f18         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f50         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f20         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f52         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f22         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f54         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f24         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f56         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f26         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f58         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f28         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f60         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f30         ;\
        ba,pt   %xcc, done2                     ;\
          ldd   [ADDR + CPU_TMP1], %f62         ;\
done2:

.lddf_exception_not_aligned:
        /* %g2 = sfar, %g3 = sfsr */
        mov     %g2, %g5                ! stash sfar
#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
        sethi   %hi(fpu_exists), %g2    ! check fpu_exists
        ld      [%g2 + %lo(fpu_exists)], %g2
        brz,a,pn %g2, 4f
          nop
#endif
        CPU_ADDR(%g1, %g4)
        or      %g0, 1, %g4
        st      %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag

        rdpr    %tpc, %g2
        lda     [%g2]ASI_AIUP, %g6      ! get the user's lddf instruction
        srl     %g6, 23, %g1            ! using ldda or not?
        and     %g1, 1, %g1
        brz,a,pt %g1, 2f                ! check for ldda instruction
          nop
        srl     %g6, 13, %g1            ! check immflag
        and     %g1, 1, %g1
        rdpr    %tstate, %g2            ! %tstate in %g2
        brnz,a,pn %g1, 1f
          srl   %g2, 31, %g1            ! get asi from %tstate
        srl     %g6, 5, %g1             ! get asi from instruction
        and     %g1, 0xFF, %g1          ! imm_asi field
1:
        cmp     %g1, ASI_P              ! primary address space
        be,a,pt %icc, 2f
          nop
        cmp     %g1, ASI_PNF            ! primary no fault address space
        be,a,pt %icc, 2f
          nop
        cmp     %g1, ASI_S              ! secondary address space
        be,a,pt %icc, 2f
          nop
        cmp     %g1, ASI_SNF            ! secondary no fault address space
        bne,a,pn %icc, 3f
          nop
2:
        lduwa   [%g5]ASI_USER, %g7      ! get first half of misaligned data
        add     %g5, 4, %g5             ! increment misaligned data address
        lduwa   [%g5]ASI_USER, %g5      ! get second half of misaligned data

        sllx    %g7, 32, %g7
        or      %g5, %g7, %g5           ! combine data
        CPU_ADDR(%g7, %g1)              ! save data on a per-cpu basis
        stx     %g5, [%g7 + CPU_TMP1]   ! save in cpu_tmp1

        srl     %g6, 25, %g3            ! %g6 has the instruction
        and     %g3, 0x1F, %g3          ! %g3 has rd
        LDDF_REG(%g3, %g7, %g4)

        CPU_ADDR(%g1, %g4)
        st      %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
        FAST_TRAP_DONE
3:
        CPU_ADDR(%g1, %g4)
        st      %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
4:
        set     T_USER, %g3             ! trap type in %g3
        or      %g3, T_LDDF_ALIGN, %g3
        mov     %g5, %g2                ! misaligned vaddr in %g2
        set     fpu_trap, %g1           ! goto C for the little and
        ba,pt   %xcc, sys_trap          ! no fault little asi's
          sub   %g0, 1, %g4

.stdf_exception_not_aligned:
        /* %g2 = sfar, %g3 = sfsr */
        mov     %g2, %g5

#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
        sethi   %hi(fpu_exists), %g7            ! check fpu_exists
        ld      [%g7 + %lo(fpu_exists)], %g3
        brz,a,pn %g3, 4f
          nop
#endif
        CPU_ADDR(%g1, %g4)
        or      %g0, 1, %g4
        st      %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag

        rdpr    %tpc, %g2
        lda     [%g2]ASI_AIUP, %g6      ! get the user's stdf instruction

        srl     %g6, 23, %g1            ! using stda or not?
        and     %g1, 1, %g1
        brz,a,pt %g1, 2f                ! check for stda instruction
          nop
        srl     %g6, 13, %g1            ! check immflag
        and     %g1, 1, %g1
        rdpr    %tstate, %g2            ! %tstate in %g2
        brnz,a,pn %g1, 1f
          srl   %g2, 31, %g1            ! get asi from %tstate
        srl     %g6, 5, %g1             ! get asi from instruction
        and     %g1, 0xff, %g1          ! imm_asi field
1:
        cmp     %g1, ASI_P              ! primary address space
        be,a,pt %icc, 2f
          nop
        cmp     %g1, ASI_S              ! secondary address space
        bne,a,pn %icc, 3f
          nop
2:
        srl     %g6, 25, %g6
        and     %g6, 0x1F, %g6          ! %g6 has rd
        CPU_ADDR(%g7, %g1)
        STDF_REG(%g6, %g7, %g4)         ! STDF_REG(REG, ADDR, TMP)

        ldx     [%g7 + CPU_TMP1], %g6
        srlx    %g6, 32, %g7
        stuwa   %g7, [%g5]ASI_USER      ! first half
        add     %g5, 4, %g5             ! increment misaligned data address
        stuwa   %g6, [%g5]ASI_USER      ! second half

        CPU_ADDR(%g1, %g4)
        st      %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
        FAST_TRAP_DONE
3:
        CPU_ADDR(%g1, %g4)
        st      %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
4:
        set     T_USER, %g3             ! trap type in %g3
        or      %g3, T_STDF_ALIGN, %g3
        mov     %g5, %g2                ! misaligned vaddr in %g2
        set     fpu_trap, %g1           ! goto C for the little and
        ba,pt   %xcc, sys_trap          ! nofault little asi's
          sub   %g0, 1, %g4

#ifdef DEBUG_USER_TRAPTRACECTL

.traptrace_freeze:
        mov     %l0, %g1 ; mov  %l1, %g2 ; mov  %l2, %g3 ; mov  %l4, %g4
        TT_TRACE_L(trace_win)
        mov     %g4, %l4 ; mov  %g3, %l2 ; mov  %g2, %l1 ; mov  %g1, %l0
        set     trap_freeze, %g1
        mov     1, %g2
        st      %g2, [%g1]
        FAST_TRAP_DONE

.traptrace_unfreeze:
        set     trap_freeze, %g1
        st      %g0, [%g1]
        mov     %l0, %g1 ; mov  %l1, %g2 ; mov  %l2, %g3 ; mov  %l4, %g4
        TT_TRACE_L(trace_win)
        mov     %g4, %l4 ; mov  %g3, %l2 ; mov  %g2, %l1 ; mov  %g1, %l0
        FAST_TRAP_DONE

#endif /* DEBUG_USER_TRAPTRACECTL */

.getcc:
        CPU_ADDR(%g1, %g2)
        stx     %o0, [%g1 + CPU_TMP1]           ! save %o0
        rdpr    %tstate, %g3                    ! get tstate
        srlx    %g3, PSR_TSTATE_CC_SHIFT, %o0   ! shift ccr to V8 psr
        set     PSR_ICC, %g2
        and     %o0, %g2, %o0                   ! mask out the rest
        srl     %o0, PSR_ICC_SHIFT, %o0         ! right justify
        wrpr    %g0, 0, %gl
        mov     %o0, %g1                        ! move ccr to normal %g1
        wrpr    %g0, 1, %gl
        ! cannot assume globals retained their values after increasing %gl
        CPU_ADDR(%g1, %g2)
        ldx     [%g1 + CPU_TMP1], %o0           ! restore %o0
        FAST_TRAP_DONE

.setcc:
        CPU_ADDR(%g1, %g2)
        stx     %o0, [%g1 + CPU_TMP1]           ! save %o0
        wrpr    %g0, 0, %gl
        mov     %g1, %o0
        wrpr    %g0, 1, %gl
        ! cannot assume globals retained their values after increasing %gl
        CPU_ADDR(%g1, %g2)
        sll     %o0, PSR_ICC_SHIFT, %g2
        set     PSR_ICC, %g3
        and     %g2, %g3, %g2                   ! mask out rest
        sllx    %g2, PSR_TSTATE_CC_SHIFT, %g2
        rdpr    %tstate, %g3                    ! get tstate
        srl     %g3, 0, %g3                     ! clear upper word
        or      %g3, %g2, %g3                   ! or in new bits
        wrpr    %g3, %tstate
        ldx     [%g1 + CPU_TMP1], %o0           ! restore %o0
        FAST_TRAP_DONE

/*
 * getpsr(void)
 * Note that the xcc part of the ccr is not provided.
 * The V8 code shows why the V9 trap is not faster:
 * #define GETPSR_TRAP() \
 *      mov %psr, %i0; jmp %l2; rett %l2+4; nop;
 */

        .type   .getpsr, #function
.getpsr:
        rdpr    %tstate, %g1                    ! get tstate
        srlx    %g1, PSR_TSTATE_CC_SHIFT, %o0   ! shift ccr to V8 psr
        set     PSR_ICC, %g2
        and     %o0, %g2, %o0                   ! mask out the rest

        rd      %fprs, %g1                      ! get fprs
        and     %g1, FPRS_FEF, %g2              ! mask out dirty upper/lower
        sllx    %g2, PSR_FPRS_FEF_SHIFT, %g2    ! shift fef to V8 psr.ef
        or      %o0, %g2, %o0                   ! or result into psr.ef

        set     V9_PSR_IMPLVER, %g2             ! SI assigned impl/ver: 0xef
        or      %o0, %g2, %o0                   ! or psr.impl/ver
        FAST_TRAP_DONE
        SET_SIZE(.getpsr)

/*
 * setpsr(newpsr)
 * Note that there is no support for ccr.xcc in the V9 code.
 */

        .type   .setpsr, #function
.setpsr:
        rdpr    %tstate, %g1                    ! get tstate
!       setx    TSTATE_V8_UBITS, %g2
        or      %g0, CCR_ICC, %g3
        sllx    %g3, TSTATE_CCR_SHIFT, %g2

        andn    %g1, %g2, %g1                   ! zero current user bits
        set     PSR_ICC, %g2
        and     %g2, %o0, %g2                   ! clear all but psr.icc bits
        sllx    %g2, PSR_TSTATE_CC_SHIFT, %g3   ! shift to tstate.ccr.icc
        wrpr    %g1, %g3, %tstate               ! write tstate

        set     PSR_EF, %g2
        and     %g2, %o0, %g2                   ! clear all but fp enable bit
        srlx    %g2, PSR_FPRS_FEF_SHIFT, %g4    ! shift ef to V9 fprs.fef
        wr      %g0, %g4, %fprs                 ! write fprs

        CPU_ADDR(%g1, %g2)                      ! load CPU struct addr to %g1
        ldn     [%g1 + CPU_THREAD], %g2         ! load thread pointer
        ldn     [%g2 + T_LWP], %g3              ! load klwp pointer
        ldn     [%g3 + LWP_FPU], %g2            ! get lwp_fpu pointer
        stuw    %g4, [%g2 + FPU_FPRS]           ! write fef value to fpu_fprs
        srlx    %g4, 2, %g4                     ! shift fef value to bit 0
        stub    %g4, [%g2 + FPU_EN]             ! write fef value to fpu_en
        FAST_TRAP_DONE
        SET_SIZE(.setpsr)

/*
 * getlgrp
 * get home lgrpid on which the calling thread is currently executing.
 */
        .type   .getlgrp, #function
.getlgrp:
        ! Thanks for the incredibly helpful comments
        CPU_ADDR(%g1, %g2)              ! load CPU struct addr to %g1 using %g2
        ld      [%g1 + CPU_ID], %o0     ! load cpu_id
        ldn     [%g1 + CPU_THREAD], %g2 ! load thread pointer
        ldn     [%g2 + T_LPL], %g2      ! load lpl pointer
        ld      [%g2 + LPL_LGRPID], %g1 ! load lpl_lgrpid
        sra     %g1, 0, %o1
        FAST_TRAP_DONE
        SET_SIZE(.getlgrp)

/*
 * Entry for old 4.x trap (trap 0).
 */
        ENTRY_NP(syscall_trap_4x)
        CPU_ADDR(%g1, %g2)              ! load CPU struct addr to %g1 using %g2
        ldn     [%g1 + CPU_THREAD], %g2 ! load thread pointer
        ldn     [%g2 + T_LWP], %g2      ! load klwp pointer
        ld      [%g2 + PCB_TRAP0], %g2  ! lwp->lwp_pcb.pcb_trap0addr
        brz,pn  %g2, 1f                 ! has it been set?
        st      %l0, [%g1 + CPU_TMP1]   ! delay - save some locals
        st      %l1, [%g1 + CPU_TMP2]
        rdpr    %tnpc, %l1              ! save old tnpc
        wrpr    %g0, %g2, %tnpc         ! setup tnpc

        mov     %g1, %l0                ! save CPU struct addr
        wrpr    %g0, 0, %gl
        mov     %l1, %g6                ! pass tnpc to user code in %g6
        wrpr    %g0, 1, %gl
        ld      [%l0 + CPU_TMP2], %l1   ! restore locals
        ld      [%l0 + CPU_TMP1], %l0
        FAST_TRAP_DONE_CHK_INTR
1:
        !
        ! check for old syscall mmap which is the only different one which
        ! must be the same.  Others are handled in the compatibility library.
        !
        mov     %g1, %l0                ! save CPU struct addr
        wrpr    %g0, 0, %gl
        cmp     %g1, OSYS_mmap          ! compare to old 4.x mmap
        movz    %icc, SYS_mmap, %g1
        wrpr    %g0, 1, %gl
        ld      [%l0 + CPU_TMP1], %l0
        SYSCALL(syscall_trap32)
        SET_SIZE(syscall_trap_4x)

/*
 * Handler for software trap 9.
 * Set trap0 emulation address for old 4.x system call trap.
 * XXX - this should be a system call.
 */
        ENTRY_NP(set_trap0_addr)
        CPU_ADDR(%g1, %g2)              ! load CPU struct addr to %g1 using %g2
        st      %l0, [%g1 + CPU_TMP1]   ! save some locals
        st      %l1, [%g1 + CPU_TMP2]
        mov     %g1, %l0        ! preserve CPU addr
        wrpr    %g0, 0, %gl
        mov     %g1, %l1
        wrpr    %g0, 1, %gl
        ! cannot assume globals retained their values after increasing %gl
        ldn     [%l0 + CPU_THREAD], %g2 ! load thread pointer
        ldn     [%g2 + T_LWP], %g2      ! load klwp pointer
        andn    %l1, 3, %l1             ! force alignment
        st      %l1, [%g2 + PCB_TRAP0]  ! lwp->lwp_pcb.pcb_trap0addr
        ld      [%l0 + CPU_TMP2], %l1   ! restore locals
        ld      [%l0 + CPU_TMP1], %l0
        FAST_TRAP_DONE
        SET_SIZE(set_trap0_addr)

/*
 * mmu_trap_tl1
 * trap handler for unexpected mmu traps.
 * simply checks if the trap was a user lddf/stdf alignment trap, in which
 * case we go to fpu_trap or a user trap from the window handler, in which
 * case we go save the state on the pcb.  Otherwise, we go to ptl1_panic.
 */
        .type   mmu_trap_tl1, #function
mmu_trap_tl1:
#ifdef  TRAPTRACE
        TRACE_PTR(%g5, %g6)
        GET_TRACE_TICK(%g6, %g7)
        stxa    %g6, [%g5 + TRAP_ENT_TICK]%asi
        TRACE_SAVE_TL_GL_REGS(%g5, %g6)
        rdpr    %tt, %g6
        stha    %g6, [%g5 + TRAP_ENT_TT]%asi
        rdpr    %tstate, %g6
        stxa    %g6, [%g5 + TRAP_ENT_TSTATE]%asi
        stna    %sp, [%g5 + TRAP_ENT_SP]%asi
        stna    %g0, [%g5 + TRAP_ENT_TR]%asi
        rdpr    %tpc, %g6
        stna    %g6, [%g5 + TRAP_ENT_TPC]%asi
        MMU_FAULT_STATUS_AREA(%g6)
        ldx     [%g6 + MMFSA_D_ADDR], %g6
        stna    %g6, [%g5 + TRAP_ENT_F1]%asi !  MMU fault address
        CPU_PADDR(%g7, %g6);
        add     %g7, CPU_TL1_HDLR, %g7
        lda     [%g7]ASI_MEM, %g6
        stna    %g6, [%g5 + TRAP_ENT_F2]%asi
        MMU_FAULT_STATUS_AREA(%g6)
        ldx     [%g6 + MMFSA_D_TYPE], %g7 ! XXXQ should be a MMFSA_F_ constant?
        ldx     [%g6 + MMFSA_D_CTX], %g6
        sllx    %g6, SFSR_CTX_SHIFT, %g6
        or      %g6, %g7, %g6
        stna    %g6, [%g5 + TRAP_ENT_F3]%asi ! MMU context/type
        set     0xdeadbeef, %g6
        stna    %g6, [%g5 + TRAP_ENT_F4]%asi
        TRACE_NEXT(%g5, %g6, %g7)
#endif /* TRAPTRACE */
        CPU_PADDR(%g7, %g6);
        add     %g7, CPU_TL1_HDLR, %g7          ! %g7 = &cpu_m.tl1_hdlr (PA)
        lda     [%g7]ASI_MEM, %g6
        brz,a,pt %g6, 1f
          nop
        sta     %g0, [%g7]ASI_MEM
        ! XXXQ need to setup registers for sfmmu_mmu_trap?
        ba,a,pt %xcc, sfmmu_mmu_trap            ! handle page faults
1:
        rdpr    %tpc, %g7
        /* in user_rtt? */
        set     rtt_fill_start, %g6
        cmp     %g7, %g6
        blu,pn  %xcc, 6f
         .empty
        set     rtt_fill_end, %g6
        cmp     %g7, %g6
        bgeu,pn %xcc, 6f
         nop
        set     fault_rtt_fn1, %g7
        ba,a    7f
6:
        ! check to see if the trap pc is in a window spill/fill handling
        rdpr    %tpc, %g7
        /* tpc should be in the trap table */
        set     trap_table, %g6
        cmp     %g7, %g6
        blu,a,pn %xcc, ptl1_panic
          mov   PTL1_BAD_MMUTRAP, %g1
        set     etrap_table, %g6
        cmp     %g7, %g6
        bgeu,a,pn %xcc, ptl1_panic
          mov   PTL1_BAD_MMUTRAP, %g1
        ! pc is inside the trap table, convert to trap type
        srl     %g7, 5, %g6             ! XXXQ need #define
        and     %g6, 0x1ff, %g6         ! XXXQ need #define
        ! and check for a window trap type
        and     %g6, WTRAP_TTMASK, %g6
        cmp     %g6, WTRAP_TYPE
        bne,a,pn %xcc, ptl1_panic
          mov   PTL1_BAD_MMUTRAP, %g1
        andn    %g7, WTRAP_ALIGN, %g7   /* 128 byte aligned */
        add     %g7, WTRAP_FAULTOFF, %g7

7:
        ! Arguments are passed in the global set active after the
        ! 'done' instruction. Before switching sets, must save
        ! the calculated next pc
        wrpr    %g0, %g7, %tnpc
        wrpr    %g0, 1, %gl
        rdpr    %tt, %g5
        MMU_FAULT_STATUS_AREA(%g7)
        cmp     %g5, T_ALIGNMENT
        be,pn   %xcc, 1f
        ldx     [%g7 + MMFSA_D_ADDR], %g6
        ldx     [%g7 + MMFSA_D_CTX], %g7
        srlx    %g6, MMU_PAGESHIFT, %g6         /* align address */
        cmp     %g7, USER_CONTEXT_TYPE
        sllx    %g6, MMU_PAGESHIFT, %g6
        movgu   %icc, USER_CONTEXT_TYPE, %g7
        or      %g6, %g7, %g6                   /* TAG_ACCESS */
1:
        done
        SET_SIZE(mmu_trap_tl1)

/*
 * Several traps use kmdb_trap and kmdb_trap_tl1 as their handlers.  These
 * traps are valid only when kmdb is loaded.  When the debugger is active,
 * the code below is rewritten to transfer control to the appropriate
 * debugger entry points.
 */
        .global kmdb_trap
        .align  8
kmdb_trap:
        ba,a    trap_table0
        jmp     %g1 + 0
        nop

        .global kmdb_trap_tl1
        .align  8
kmdb_trap_tl1:
        ba,a    trap_table0
        jmp     %g1 + 0
        nop

/*
 * This entry is copied from OBP's trap table during boot.
 */
        .global obp_bpt
        .align  8
obp_bpt:
        NOT



#ifdef  TRAPTRACE
/*
 * TRAPTRACE support.
 * labels here are branched to with "rd %pc, %g7" in the delay slot.
 * Return is done by "jmp %g7 + 4".
 */

trace_dmmu:
        TRACE_PTR(%g3, %g6)
        GET_TRACE_TICK(%g6, %g5)
        stxa    %g6, [%g3 + TRAP_ENT_TICK]%asi
        TRACE_SAVE_TL_GL_REGS(%g3, %g6)
        rdpr    %tt, %g6
        stha    %g6, [%g3 + TRAP_ENT_TT]%asi
        rdpr    %tstate, %g6
        stxa    %g6, [%g3 + TRAP_ENT_TSTATE]%asi
        stna    %sp, [%g3 + TRAP_ENT_SP]%asi
        rdpr    %tpc, %g6
        stna    %g6, [%g3 + TRAP_ENT_TPC]%asi
        MMU_FAULT_STATUS_AREA(%g6)
        ldx     [%g6 + MMFSA_D_ADDR], %g4
        stxa    %g4, [%g3 + TRAP_ENT_TR]%asi
        ldx     [%g6 + MMFSA_D_CTX], %g4
        stxa    %g4, [%g3 + TRAP_ENT_F1]%asi
        ldx     [%g6 + MMFSA_D_TYPE], %g4
        stxa    %g4, [%g3 + TRAP_ENT_F2]%asi
        stxa    %g6, [%g3 + TRAP_ENT_F3]%asi
        stna    %g0, [%g3 + TRAP_ENT_F4]%asi
        TRACE_NEXT(%g3, %g4, %g5)
        jmp     %g7 + 4
        nop

trace_immu:
        TRACE_PTR(%g3, %g6)
        GET_TRACE_TICK(%g6, %g5)
        stxa    %g6, [%g3 + TRAP_ENT_TICK]%asi
        TRACE_SAVE_TL_GL_REGS(%g3, %g6)
        rdpr    %tt, %g6
        stha    %g6, [%g3 + TRAP_ENT_TT]%asi
        rdpr    %tstate, %g6
        stxa    %g6, [%g3 + TRAP_ENT_TSTATE]%asi
        stna    %sp, [%g3 + TRAP_ENT_SP]%asi
        rdpr    %tpc, %g6
        stna    %g6, [%g3 + TRAP_ENT_TPC]%asi
        MMU_FAULT_STATUS_AREA(%g6)
        ldx     [%g6 + MMFSA_I_ADDR], %g4
        stxa    %g4, [%g3 + TRAP_ENT_TR]%asi
        ldx     [%g6 + MMFSA_I_CTX], %g4
        stxa    %g4, [%g3 + TRAP_ENT_F1]%asi
        ldx     [%g6 + MMFSA_I_TYPE], %g4
        stxa    %g4, [%g3 + TRAP_ENT_F2]%asi
        stxa    %g6, [%g3 + TRAP_ENT_F3]%asi
        stna    %g0, [%g3 + TRAP_ENT_F4]%asi
        TRACE_NEXT(%g3, %g4, %g5)
        jmp     %g7 + 4
        nop

trace_gen:
        TRACE_PTR(%g3, %g6)
        GET_TRACE_TICK(%g6, %g5)
        stxa    %g6, [%g3 + TRAP_ENT_TICK]%asi
        TRACE_SAVE_TL_GL_REGS(%g3, %g6)
        rdpr    %tt, %g6
        stha    %g6, [%g3 + TRAP_ENT_TT]%asi
        rdpr    %tstate, %g6
        stxa    %g6, [%g3 + TRAP_ENT_TSTATE]%asi
        stna    %sp, [%g3 + TRAP_ENT_SP]%asi
        rdpr    %tpc, %g6
        stna    %g6, [%g3 + TRAP_ENT_TPC]%asi
        stna    %g0, [%g3 + TRAP_ENT_TR]%asi
        stna    %g0, [%g3 + TRAP_ENT_F1]%asi
        stna    %g0, [%g3 + TRAP_ENT_F2]%asi
        stna    %g0, [%g3 + TRAP_ENT_F3]%asi
        stna    %g0, [%g3 + TRAP_ENT_F4]%asi
        TRACE_NEXT(%g3, %g4, %g5)
        jmp     %g7 + 4
        nop

trace_win:
        TRACE_WIN_INFO(0, %l0, %l1, %l2)
        ! Keep the locals as clean as possible, caller cleans %l4
        clr     %l2
        clr     %l1
        jmp     %l4 + 4
          clr   %l0

/*
 * Trace a tsb hit
 * g1 = tsbe pointer (in/clobbered)
 * g2 = tag access register (in)
 * g3 - g4 = scratch (clobbered)
 * g5 = tsbe data (in)
 * g6 = scratch (clobbered)
 * g7 = pc we jumped here from (in)
 */

        ! Do not disturb %g5, it will be used after the trace
        ALTENTRY(trace_tsbhit)
        TRACE_TSBHIT(0)
        jmp     %g7 + 4
        nop

/*
 * Trace a TSB miss
 *
 * g1 = tsb8k pointer (in)
 * g2 = tag access register (in)
 * g3 = tsb4m pointer (in)
 * g4 = tsbe tag (in/clobbered)
 * g5 - g6 = scratch (clobbered)
 * g7 = pc we jumped here from (in)
 */
        .global trace_tsbmiss
trace_tsbmiss:
        membar  #Sync
        sethi   %hi(FLUSH_ADDR), %g6
        flush   %g6
        TRACE_PTR(%g5, %g6)
        stna    %g2, [%g5 + TRAP_ENT_SP]%asi            ! tag access
        stna    %g4, [%g5 + TRAP_ENT_F1]%asi            ! XXX? tsb tag
        GET_TRACE_TICK(%g6, %g4)
        stxa    %g6, [%g5 + TRAP_ENT_TICK]%asi
        rdpr    %tnpc, %g6
        stna    %g6, [%g5 + TRAP_ENT_F2]%asi
        stna    %g1, [%g5 + TRAP_ENT_F3]%asi            ! tsb8k pointer
        rdpr    %tpc, %g6
        stna    %g6, [%g5 + TRAP_ENT_TPC]%asi
        TRACE_SAVE_TL_GL_REGS(%g5, %g6)
        rdpr    %tt, %g6
        or      %g6, TT_MMU_MISS, %g4
        stha    %g4, [%g5 + TRAP_ENT_TT]%asi
        mov     MMFSA_D_ADDR, %g4
        cmp     %g6, FAST_IMMU_MISS_TT
        move    %xcc, MMFSA_I_ADDR, %g4
        cmp     %g6, T_INSTR_MMU_MISS
        move    %xcc, MMFSA_I_ADDR, %g4
        MMU_FAULT_STATUS_AREA(%g6)
        ldx     [%g6 + %g4], %g6
        stxa    %g6, [%g5 + TRAP_ENT_TSTATE]%asi        ! tag target
        cmp     %g4, MMFSA_D_ADDR
        move    %xcc, MMFSA_D_CTX, %g4
        movne   %xcc, MMFSA_I_CTX, %g4
        MMU_FAULT_STATUS_AREA(%g6)
        ldx     [%g6 + %g4], %g6
        stxa    %g6, [%g5 + TRAP_ENT_F4]%asi            ! context ID
        stna    %g3, [%g5 + TRAP_ENT_TR]%asi            ! tsb4m pointer
        TRACE_NEXT(%g5, %g4, %g6)
        jmp     %g7 + 4
        nop

/*
 * g2 = tag access register (in)
 * g3 = ctx type (0, 1 or 2) (in) (not used)
 */
trace_dataprot:
        membar  #Sync
        sethi   %hi(FLUSH_ADDR), %g6
        flush   %g6
        TRACE_PTR(%g1, %g6)
        GET_TRACE_TICK(%g6, %g4)
        stxa    %g6, [%g1 + TRAP_ENT_TICK]%asi
        rdpr    %tpc, %g6
        stna    %g6, [%g1 + TRAP_ENT_TPC]%asi
        rdpr    %tstate, %g6
        stxa    %g6, [%g1 + TRAP_ENT_TSTATE]%asi
        stna    %g2, [%g1 + TRAP_ENT_SP]%asi            ! tag access reg
        stna    %g0, [%g1 + TRAP_ENT_F1]%asi
        stna    %g0, [%g1 + TRAP_ENT_F2]%asi
        stna    %g0, [%g1 + TRAP_ENT_F3]%asi
        stna    %g0, [%g1 + TRAP_ENT_F4]%asi
        TRACE_SAVE_TL_GL_REGS(%g1, %g6)
        rdpr    %tt, %g6
        stha    %g6, [%g1 + TRAP_ENT_TT]%asi
        mov     MMFSA_D_CTX, %g4
        cmp     %g6, FAST_IMMU_MISS_TT
        move    %xcc, MMFSA_I_CTX, %g4
        cmp     %g6, T_INSTR_MMU_MISS
        move    %xcc, MMFSA_I_CTX, %g4
        MMU_FAULT_STATUS_AREA(%g6)
        ldx     [%g6 + %g4], %g6
        stxa    %g6, [%g1 + TRAP_ENT_TR]%asi    ! context ID
        TRACE_NEXT(%g1, %g4, %g5)
        jmp     %g7 + 4
        nop

#endif /* TRAPTRACE */

/*
 * Handle watchdog reset trap. Enable the MMU using the MMU_ENABLE
 * HV service, which requires the return target to be specified as a VA
 * since we are enabling the MMU. We set the target to ptl1_panic.
 */

        .type   .watchdog_trap, #function
.watchdog_trap:
        mov     1, %o0
        setx    ptl1_panic, %g2, %o1
        mov     MMU_ENABLE, %o5
        ta      FAST_TRAP
        done
        SET_SIZE(.watchdog_trap)
/*
 * synthesize for trap(): SFAR in %g2, SFSR in %g3
 */
        .type   .dmmu_exc_lddf_not_aligned, #function
.dmmu_exc_lddf_not_aligned:
        MMU_FAULT_STATUS_AREA(%g3)
        ldx     [%g3 + MMFSA_D_ADDR], %g2
        /* Fault type not available in MMU fault status area */
        mov     MMFSA_F_UNALIGN, %g1
        ldx     [%g3 + MMFSA_D_CTX], %g3
        sllx    %g3, SFSR_CTX_SHIFT, %g3
        btst    1, %sp
        bnz,pt  %xcc, .lddf_exception_not_aligned
        or      %g3, %g1, %g3                   /* SFSR */
        ba,a,pt %xcc, .mmu_exception_not_aligned
        SET_SIZE(.dmmu_exc_lddf_not_aligned)

/*
 * synthesize for trap(): SFAR in %g2, SFSR in %g3
 */
        .type   .dmmu_exc_stdf_not_aligned, #function
.dmmu_exc_stdf_not_aligned:
        MMU_FAULT_STATUS_AREA(%g3)
        ldx     [%g3 + MMFSA_D_ADDR], %g2
        /* Fault type not available in MMU fault status area */
        mov     MMFSA_F_UNALIGN, %g1
        ldx     [%g3 + MMFSA_D_CTX], %g3
        sllx    %g3, SFSR_CTX_SHIFT, %g3
        btst    1, %sp
        bnz,pt  %xcc, .stdf_exception_not_aligned
        or      %g3, %g1, %g3                   /* SFSR */
        ba,a,pt %xcc, .mmu_exception_not_aligned
        SET_SIZE(.dmmu_exc_stdf_not_aligned)

        .type   .dmmu_exception, #function
.dmmu_exception:
        MMU_FAULT_STATUS_AREA(%g3)
        ldx     [%g3 + MMFSA_D_ADDR], %g2
        ldx     [%g3 + MMFSA_D_TYPE], %g1
        ldx     [%g3 + MMFSA_D_CTX], %g4
        srlx    %g2, MMU_PAGESHIFT, %g2         /* align address */
        sllx    %g2, MMU_PAGESHIFT, %g2
        sllx    %g4, SFSR_CTX_SHIFT, %g3
        or      %g3, %g1, %g3                   /* SFSR */
        cmp     %g4, USER_CONTEXT_TYPE
        movgeu  %icc, USER_CONTEXT_TYPE, %g4
        or      %g2, %g4, %g2                   /* TAG_ACCESS */
        ba,pt   %xcc, .mmu_exception_end
        mov     T_DATA_EXCEPTION, %g1
        SET_SIZE(.dmmu_exception)

        .align 32
        .global pil15_epilogue
pil15_epilogue:
        ba      pil_interrupt_common
        nop
        .align 32

/*
 * fast_trap_done, fast_trap_done_chk_intr:
 *
 * Due to the design of UltraSPARC pipeline, pending interrupts are not
 * taken immediately after a RETRY or DONE instruction which causes IE to
 * go from 0 to 1. Instead, the instruction at %tpc or %tnpc is allowed
 * to execute first before taking any interrupts. If that instruction
 * results in other traps, and if the corresponding trap handler runs
 * entirely at TL=1 with interrupts disabled, then pending interrupts
 * won't be taken until after yet another instruction following the %tpc
 * or %tnpc.
 *
 * A malicious user program can use this feature to block out interrupts
 * for extended durations, which can result in send_mondo_timeout kernel
 * panic.
 *
 * This problem is addressed by servicing any pending interrupts via
 * sys_trap before returning back to the user mode from a fast trap
 * handler. The "done" instruction within a fast trap handler, which
 * runs entirely at TL=1 with interrupts disabled, is replaced with the
 * FAST_TRAP_DONE macro, which branches control to this fast_trap_done
 * entry point.
 *
 * We check for any pending interrupts here and force a sys_trap to
 * service those interrupts, if any. To minimize overhead, pending
 * interrupts are checked if the %tpc happens to be at 16K boundary,
 * which allows a malicious program to execute at most 4K consecutive
 * instructions before we service any pending interrupts. If a worst
 * case fast trap handler takes about 2 usec, then interrupts will be
 * blocked for at most 8 msec, less than a clock tick.
 *
 * For the cases where we don't know if the %tpc will cross a 16K
 * boundary, we can't use the above optimization and always process
 * any pending interrupts via fast_frap_done_chk_intr entry point.
 *
 * Entry Conditions:
 *      %pstate         am:0 priv:1 ie:0
 *                      globals are AG (not normal globals)
 */

        .global fast_trap_done, fast_trap_done_chk_intr
fast_trap_done:
        rdpr    %tpc, %g5
        sethi   %hi(0xffffc000), %g6    ! 1's complement of 0x3fff
        andncc  %g5, %g6, %g0           ! check lower 14 bits of %tpc
        bz,pn   %icc, 1f                ! branch if zero (lower 32 bits only)
        nop
        done

fast_trap_done_chk_intr:
1:      rd      SOFTINT, %g6
        brnz,pn %g6, 2f         ! branch if any pending intr
        nop
        done

2:
        /*
         * We get here if there are any pending interrupts.
         * Adjust %tpc/%tnpc as we'll be resuming via "retry"
         * instruction.
         */
        rdpr    %tnpc, %g5
        wrpr    %g0, %g5, %tpc
        add     %g5, 4, %g5
        wrpr    %g0, %g5, %tnpc

        /*
         * Force a dummy sys_trap call so that interrupts can be serviced.
         */
        set     fast_trap_dummy_call, %g1
        ba,pt   %xcc, sys_trap
          mov   -1, %g4

fast_trap_dummy_call:
        retl
        nop

/*
 * Currently the brand syscall interposition code is not enabled by
 * default.  Instead, when a branded zone is first booted the brand
 * infrastructure will patch the trap table so that the syscall
 * entry points are redirected to syscall_wrapper32 and syscall_wrapper
 * for ILP32 and LP64 syscalls respectively.  this is done in
 * brand_plat_interposition_enable().  Note that the syscall wrappers
 * below do not collect any trap trace data since the syscall hot patch
 * points are reached after trap trace data has already been collected.
 */
#define BRAND_CALLBACK(callback_id)                                         \
        CPU_ADDR(%g2, %g1)              /* load CPU struct addr to %g2  */ ;\
        ldn     [%g2 + CPU_THREAD], %g3 /* load thread pointer          */ ;\
        ldn     [%g3 + T_PROCP], %g3    /* get proc pointer             */ ;\
        ldn     [%g3 + P_BRAND], %g3    /* get brand pointer            */ ;\
        brz     %g3, 1f                 /* No brand?  No callback.      */ ;\
        nop                                                                ;\
        ldn     [%g3 + B_MACHOPS], %g3  /* get machops list             */ ;\
        ldn     [%g3 + (callback_id << 3)], %g3                            ;\
        brz     %g3, 1f                                                    ;\
        /*                                                                  \
         * This isn't pretty.  We want a low-latency way for the callback   \
         * routine to decline to do anything.  We just pass in an address   \
         * the routine can directly jmp back to, pretending that nothing    \
         * has happened.                                                    \
         *                                                                  \
         * %g1: return address (where the brand handler jumps back to)      \
         * %g2: address of CPU structure                                    \
         * %g3: address of brand handler (where we will jump to)            \
         */                                                                 \
        mov     %pc, %g1                                                   ;\
        add     %g1, 16, %g1                                               ;\
        jmp     %g3                                                        ;\
        nop                                                                ;\
1:

        ENTRY_NP(syscall_wrapper32)
        BRAND_CALLBACK(BRAND_CB_SYSCALL32)
        SYSCALL_NOTT(syscall_trap32)
        SET_SIZE(syscall_wrapper32)

        ENTRY_NP(syscall_wrapper)
        BRAND_CALLBACK(BRAND_CB_SYSCALL)
        SYSCALL_NOTT(syscall_trap)
        SET_SIZE(syscall_wrapper)