#include "assym.h"
#include <sys/asm_linkage.h>
#include <sys/privregs.h>
#include <sys/sun4asi.h>
#include <sys/spitregs.h>
#include <sys/cheetahregs.h>
#include <sys/machtrap.h>
#include <sys/machthread.h>
#include <sys/machbrand.h>
#include <sys/pcb.h>
#include <sys/pte.h>
#include <sys/mmu.h>
#include <sys/machpcb.h>
#include <sys/async.h>
#include <sys/intreg.h>
#include <sys/scb.h>
#include <sys/psr_compat.h>
#include <sys/syscall.h>
#include <sys/machparam.h>
#include <sys/traptrace.h>
#include <vm/hat_sfmmu.h>
#include <sys/archsystm.h>
#include <sys/utrap.h>
#include <sys/clock.h>
#include <sys/intr.h>
#include <sys/fpu/fpu_simulator.h>
#include <vm/seg_spt.h>
#define FAST_TRAP_DONE \
ba,a fast_trap_done
#define FAST_TRAP_DONE_CHK_INTR \
ba,a fast_trap_done_chk_intr
#ifdef TRAPTRACE
#define TT_TRACE(label) \
ba label ;\
rd %pc, %g7
#define TT_TRACE_INS 2
#define TT_TRACE_L(label) \
ba label ;\
rd %pc, %l4 ;\
clr %l4
#define TT_TRACE_L_INS 3
#else
#define TT_TRACE(label)
#define TT_TRACE_INS 0
#define TT_TRACE_L(label)
#define TT_TRACE_L_INS 0
#endif
.global trap
#define NOT \
TT_TRACE(trace_gen) ;\
set trap, %g1 ;\
rdpr %tt, %g3 ;\
ba,pt %xcc, sys_trap ;\
sub %g0, 1, %g4 ;\
.align 32
#define NOT4 NOT; NOT; NOT; NOT
#define RED NOT
#define BAD NOT
#define BAD4 NOT4
#define DONE \
done; \
.align 32
.global trap
#define TRAP(arg) \
TT_TRACE(trace_gen) ;\
set trap, %g1 ;\
mov arg, %g3 ;\
ba,pt %xcc, sys_trap ;\
sub %g0, 1, %g4 ;\
.align 32
#define SYSCALL_NOTT(which) \
set (which), %g1 ;\
ba,pt %xcc, sys_trap ;\
sub %g0, 1, %g4 ;\
.align 32
#define SYSCALL(which) \
TT_TRACE(trace_gen) ;\
SYSCALL_NOTT(which)
#define SYSCALL_TRAP32 \
TT_TRACE(trace_gen) ;\
ALTENTRY(syscall_trap32_patch_point) \
SYSCALL_NOTT(syscall_trap32)
#define SYSCALL_TRAP \
TT_TRACE(trace_gen) ;\
ALTENTRY(syscall_trap_patch_point) \
SYSCALL_NOTT(syscall_trap)
#define FLUSHW(h_name) \
.global h_name ;\
h_name: ;\
set trap, %g1 ;\
mov T_FLUSHW, %g3 ;\
sub %g0, 1, %g4 ;\
save ;\
flushw ;\
restore ;\
FAST_TRAP_DONE ;\
.align 32
#define GOTO(label) \
.global label ;\
ba,a label ;\
.empty ;\
.align 32
#define GOTO_TT(label, ttlabel) \
.global label ;\
TT_TRACE(ttlabel) ;\
ba,a label ;\
.empty ;\
.align 32
#define PRIV(label) \
rdpr %tstate, %g1 ;\
btst TSTATE_PRIV, %g1 ;\
bnz label ;\
rdpr %tt, %g3 ;\
set trap, %g1 ;\
ba,pt %xcc, sys_trap ;\
sub %g0, 1, %g4 ;\
.align 32
#define DTRACE_PID \
.global dtrace_pid_probe ;\
set dtrace_pid_probe, %g1 ;\
ba,pt %xcc, user_trap ;\
sub %g0, 1, %g4 ;\
.align 32
#define DTRACE_RETURN \
.global dtrace_return_probe ;\
set dtrace_return_probe, %g1 ;\
ba,pt %xcc, user_trap ;\
sub %g0, 1, %g4 ;\
.align 32
#define SKIP(n) .skip 4*(n)
#define CLEAN_WINDOW \
TT_TRACE_L(trace_win) ;\
rdpr %cleanwin, %l0; inc %l0; wrpr %l0, %cleanwin ;\
clr %l0; clr %l1; clr %l2; clr %l3 ;\
clr %l4; clr %l5; clr %l6; clr %l7 ;\
clr %o0; clr %o1; clr %o2; clr %o3 ;\
clr %o4; clr %o5; clr %o6; clr %o7 ;\
retry; .align 128
#define SPILL_32bit(tail) \
srl %sp, 0, %sp ;\
1: st %l0, [%sp + 0] ;\
st %l1, [%sp + 4] ;\
st %l2, [%sp + 8] ;\
st %l3, [%sp + 12] ;\
st %l4, [%sp + 16] ;\
st %l5, [%sp + 20] ;\
st %l6, [%sp + 24] ;\
st %l7, [%sp + 28] ;\
st %i0, [%sp + 32] ;\
st %i1, [%sp + 36] ;\
st %i2, [%sp + 40] ;\
st %i3, [%sp + 44] ;\
st %i4, [%sp + 48] ;\
st %i5, [%sp + 52] ;\
st %i6, [%sp + 56] ;\
st %i7, [%sp + 60] ;\
TT_TRACE_L(trace_win) ;\
saved ;\
retry ;\
SKIP(31-19-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_32bit_##tail ;\
.empty
#define SPILL_32bit_asi(asi_num, tail) \
srl %sp, 0, %sp ;\
1: sta %l0, [%sp + %g0]asi_num ;\
mov 4, %g1 ;\
sta %l1, [%sp + %g1]asi_num ;\
mov 8, %g2 ;\
sta %l2, [%sp + %g2]asi_num ;\
mov 12, %g3 ;\
sta %l3, [%sp + %g3]asi_num ;\
add %sp, 16, %g4 ;\
sta %l4, [%g4 + %g0]asi_num ;\
sta %l5, [%g4 + %g1]asi_num ;\
sta %l6, [%g4 + %g2]asi_num ;\
sta %l7, [%g4 + %g3]asi_num ;\
add %g4, 16, %g4 ;\
sta %i0, [%g4 + %g0]asi_num ;\
sta %i1, [%g4 + %g1]asi_num ;\
sta %i2, [%g4 + %g2]asi_num ;\
sta %i3, [%g4 + %g3]asi_num ;\
add %g4, 16, %g4 ;\
sta %i4, [%g4 + %g0]asi_num ;\
sta %i5, [%g4 + %g1]asi_num ;\
sta %i6, [%g4 + %g2]asi_num ;\
sta %i7, [%g4 + %g3]asi_num ;\
TT_TRACE_L(trace_win) ;\
saved ;\
retry ;\
SKIP(31-25-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_32bit_##tail ;\
.empty
#define SPILL_32bit_tt1(asi_num, tail) \
mov asi_num, %asi ;\
1: srl %sp, 0, %sp ;\
sta %l0, [%sp + 0]%asi ;\
sta %l1, [%sp + 4]%asi ;\
sta %l2, [%sp + 8]%asi ;\
sta %l3, [%sp + 12]%asi ;\
sta %l4, [%sp + 16]%asi ;\
sta %l5, [%sp + 20]%asi ;\
sta %l6, [%sp + 24]%asi ;\
sta %l7, [%sp + 28]%asi ;\
sta %i0, [%sp + 32]%asi ;\
sta %i1, [%sp + 36]%asi ;\
sta %i2, [%sp + 40]%asi ;\
sta %i3, [%sp + 44]%asi ;\
sta %i4, [%sp + 48]%asi ;\
sta %i5, [%sp + 52]%asi ;\
sta %i6, [%sp + 56]%asi ;\
sta %i7, [%sp + 60]%asi ;\
TT_TRACE_L(trace_win) ;\
saved ;\
retry ;\
SKIP(31-20-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_32bit_##tail ;\
.empty
#define FILL_32bit(tail) \
srl %sp, 0, %sp ;\
1: TT_TRACE_L(trace_win) ;\
ld [%sp + 0], %l0 ;\
ld [%sp + 4], %l1 ;\
ld [%sp + 8], %l2 ;\
ld [%sp + 12], %l3 ;\
ld [%sp + 16], %l4 ;\
ld [%sp + 20], %l5 ;\
ld [%sp + 24], %l6 ;\
ld [%sp + 28], %l7 ;\
ld [%sp + 32], %i0 ;\
ld [%sp + 36], %i1 ;\
ld [%sp + 40], %i2 ;\
ld [%sp + 44], %i3 ;\
ld [%sp + 48], %i4 ;\
ld [%sp + 52], %i5 ;\
ld [%sp + 56], %i6 ;\
ld [%sp + 60], %i7 ;\
restored ;\
retry ;\
SKIP(31-19-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_32bit_##tail ;\
.empty
#define FILL_32bit_asi(asi_num, tail) \
srl %sp, 0, %sp ;\
1: TT_TRACE_L(trace_win) ;\
mov 4, %g1 ;\
lda [%sp + %g0]asi_num, %l0 ;\
mov 8, %g2 ;\
lda [%sp + %g1]asi_num, %l1 ;\
mov 12, %g3 ;\
lda [%sp + %g2]asi_num, %l2 ;\
lda [%sp + %g3]asi_num, %l3 ;\
add %sp, 16, %g4 ;\
lda [%g4 + %g0]asi_num, %l4 ;\
lda [%g4 + %g1]asi_num, %l5 ;\
lda [%g4 + %g2]asi_num, %l6 ;\
lda [%g4 + %g3]asi_num, %l7 ;\
add %g4, 16, %g4 ;\
lda [%g4 + %g0]asi_num, %i0 ;\
lda [%g4 + %g1]asi_num, %i1 ;\
lda [%g4 + %g2]asi_num, %i2 ;\
lda [%g4 + %g3]asi_num, %i3 ;\
add %g4, 16, %g4 ;\
lda [%g4 + %g0]asi_num, %i4 ;\
lda [%g4 + %g1]asi_num, %i5 ;\
lda [%g4 + %g2]asi_num, %i6 ;\
lda [%g4 + %g3]asi_num, %i7 ;\
restored ;\
retry ;\
SKIP(31-25-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_32bit_##tail ;\
.empty
#define FILL_32bit_tt1(asi_num, tail) \
mov asi_num, %asi ;\
1: srl %sp, 0, %sp ;\
TT_TRACE_L(trace_win) ;\
lda [%sp + 0]%asi, %l0 ;\
lda [%sp + 4]%asi, %l1 ;\
lda [%sp + 8]%asi, %l2 ;\
lda [%sp + 12]%asi, %l3 ;\
lda [%sp + 16]%asi, %l4 ;\
lda [%sp + 20]%asi, %l5 ;\
lda [%sp + 24]%asi, %l6 ;\
lda [%sp + 28]%asi, %l7 ;\
lda [%sp + 32]%asi, %i0 ;\
lda [%sp + 36]%asi, %i1 ;\
lda [%sp + 40]%asi, %i2 ;\
lda [%sp + 44]%asi, %i3 ;\
lda [%sp + 48]%asi, %i4 ;\
lda [%sp + 52]%asi, %i5 ;\
lda [%sp + 56]%asi, %i6 ;\
lda [%sp + 60]%asi, %i7 ;\
restored ;\
retry ;\
SKIP(31-20-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_32bit_##tail ;\
.empty
#define SPILL_64bit(tail) \
2: stx %l0, [%sp + V9BIAS64 + 0] ;\
stx %l1, [%sp + V9BIAS64 + 8] ;\
stx %l2, [%sp + V9BIAS64 + 16] ;\
stx %l3, [%sp + V9BIAS64 + 24] ;\
stx %l4, [%sp + V9BIAS64 + 32] ;\
stx %l5, [%sp + V9BIAS64 + 40] ;\
stx %l6, [%sp + V9BIAS64 + 48] ;\
stx %l7, [%sp + V9BIAS64 + 56] ;\
stx %i0, [%sp + V9BIAS64 + 64] ;\
stx %i1, [%sp + V9BIAS64 + 72] ;\
stx %i2, [%sp + V9BIAS64 + 80] ;\
stx %i3, [%sp + V9BIAS64 + 88] ;\
stx %i4, [%sp + V9BIAS64 + 96] ;\
stx %i5, [%sp + V9BIAS64 + 104] ;\
stx %i6, [%sp + V9BIAS64 + 112] ;\
stx %i7, [%sp + V9BIAS64 + 120] ;\
TT_TRACE_L(trace_win) ;\
saved ;\
retry ;\
SKIP(31-18-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_64bit_##tail ;\
.empty
#define SPILL_64bit_asi(asi_num, tail) \
mov 0 + V9BIAS64, %g1 ;\
2: stxa %l0, [%sp + %g1]asi_num ;\
mov 8 + V9BIAS64, %g2 ;\
stxa %l1, [%sp + %g2]asi_num ;\
mov 16 + V9BIAS64, %g3 ;\
stxa %l2, [%sp + %g3]asi_num ;\
mov 24 + V9BIAS64, %g4 ;\
stxa %l3, [%sp + %g4]asi_num ;\
add %sp, 32, %g5 ;\
stxa %l4, [%g5 + %g1]asi_num ;\
stxa %l5, [%g5 + %g2]asi_num ;\
stxa %l6, [%g5 + %g3]asi_num ;\
stxa %l7, [%g5 + %g4]asi_num ;\
add %g5, 32, %g5 ;\
stxa %i0, [%g5 + %g1]asi_num ;\
stxa %i1, [%g5 + %g2]asi_num ;\
stxa %i2, [%g5 + %g3]asi_num ;\
stxa %i3, [%g5 + %g4]asi_num ;\
add %g5, 32, %g5 ;\
stxa %i4, [%g5 + %g1]asi_num ;\
stxa %i5, [%g5 + %g2]asi_num ;\
stxa %i6, [%g5 + %g3]asi_num ;\
stxa %i7, [%g5 + %g4]asi_num ;\
TT_TRACE_L(trace_win) ;\
saved ;\
retry ;\
SKIP(31-25-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_64bit_##tail ;\
.empty
#define SPILL_64bit_tt1(asi_num, tail) \
mov asi_num, %asi ;\
2: stxa %l0, [%sp + V9BIAS64 + 0]%asi ;\
stxa %l1, [%sp + V9BIAS64 + 8]%asi ;\
stxa %l2, [%sp + V9BIAS64 + 16]%asi ;\
stxa %l3, [%sp + V9BIAS64 + 24]%asi ;\
stxa %l4, [%sp + V9BIAS64 + 32]%asi ;\
stxa %l5, [%sp + V9BIAS64 + 40]%asi ;\
stxa %l6, [%sp + V9BIAS64 + 48]%asi ;\
stxa %l7, [%sp + V9BIAS64 + 56]%asi ;\
stxa %i0, [%sp + V9BIAS64 + 64]%asi ;\
stxa %i1, [%sp + V9BIAS64 + 72]%asi ;\
stxa %i2, [%sp + V9BIAS64 + 80]%asi ;\
stxa %i3, [%sp + V9BIAS64 + 88]%asi ;\
stxa %i4, [%sp + V9BIAS64 + 96]%asi ;\
stxa %i5, [%sp + V9BIAS64 + 104]%asi ;\
stxa %i6, [%sp + V9BIAS64 + 112]%asi ;\
stxa %i7, [%sp + V9BIAS64 + 120]%asi ;\
TT_TRACE_L(trace_win) ;\
saved ;\
retry ;\
SKIP(31-19-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_64bit_##tail ;\
.empty
#define FILL_64bit(tail) \
2: TT_TRACE_L(trace_win) ;\
ldx [%sp + V9BIAS64 + 0], %l0 ;\
ldx [%sp + V9BIAS64 + 8], %l1 ;\
ldx [%sp + V9BIAS64 + 16], %l2 ;\
ldx [%sp + V9BIAS64 + 24], %l3 ;\
ldx [%sp + V9BIAS64 + 32], %l4 ;\
ldx [%sp + V9BIAS64 + 40], %l5 ;\
ldx [%sp + V9BIAS64 + 48], %l6 ;\
ldx [%sp + V9BIAS64 + 56], %l7 ;\
ldx [%sp + V9BIAS64 + 64], %i0 ;\
ldx [%sp + V9BIAS64 + 72], %i1 ;\
ldx [%sp + V9BIAS64 + 80], %i2 ;\
ldx [%sp + V9BIAS64 + 88], %i3 ;\
ldx [%sp + V9BIAS64 + 96], %i4 ;\
ldx [%sp + V9BIAS64 + 104], %i5 ;\
ldx [%sp + V9BIAS64 + 112], %i6 ;\
ldx [%sp + V9BIAS64 + 120], %i7 ;\
restored ;\
retry ;\
SKIP(31-18-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_64bit_##tail ;\
.empty
#define FILL_64bit_asi(asi_num, tail) \
mov V9BIAS64 + 0, %g1 ;\
2: TT_TRACE_L(trace_win) ;\
ldxa [%sp + %g1]asi_num, %l0 ;\
mov V9BIAS64 + 8, %g2 ;\
ldxa [%sp + %g2]asi_num, %l1 ;\
mov V9BIAS64 + 16, %g3 ;\
ldxa [%sp + %g3]asi_num, %l2 ;\
mov V9BIAS64 + 24, %g4 ;\
ldxa [%sp + %g4]asi_num, %l3 ;\
add %sp, 32, %g5 ;\
ldxa [%g5 + %g1]asi_num, %l4 ;\
ldxa [%g5 + %g2]asi_num, %l5 ;\
ldxa [%g5 + %g3]asi_num, %l6 ;\
ldxa [%g5 + %g4]asi_num, %l7 ;\
add %g5, 32, %g5 ;\
ldxa [%g5 + %g1]asi_num, %i0 ;\
ldxa [%g5 + %g2]asi_num, %i1 ;\
ldxa [%g5 + %g3]asi_num, %i2 ;\
ldxa [%g5 + %g4]asi_num, %i3 ;\
add %g5, 32, %g5 ;\
ldxa [%g5 + %g1]asi_num, %i4 ;\
ldxa [%g5 + %g2]asi_num, %i5 ;\
ldxa [%g5 + %g3]asi_num, %i6 ;\
ldxa [%g5 + %g4]asi_num, %i7 ;\
restored ;\
retry ;\
SKIP(31-25-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_64bit_##tail ;\
.empty
#define FILL_64bit_tt1(asi_num, tail) \
mov asi_num, %asi ;\
TT_TRACE_L(trace_win) ;\
ldxa [%sp + V9BIAS64 + 0]%asi, %l0 ;\
ldxa [%sp + V9BIAS64 + 8]%asi, %l1 ;\
ldxa [%sp + V9BIAS64 + 16]%asi, %l2 ;\
ldxa [%sp + V9BIAS64 + 24]%asi, %l3 ;\
ldxa [%sp + V9BIAS64 + 32]%asi, %l4 ;\
ldxa [%sp + V9BIAS64 + 40]%asi, %l5 ;\
ldxa [%sp + V9BIAS64 + 48]%asi, %l6 ;\
ldxa [%sp + V9BIAS64 + 56]%asi, %l7 ;\
ldxa [%sp + V9BIAS64 + 64]%asi, %i0 ;\
ldxa [%sp + V9BIAS64 + 72]%asi, %i1 ;\
ldxa [%sp + V9BIAS64 + 80]%asi, %i2 ;\
ldxa [%sp + V9BIAS64 + 88]%asi, %i3 ;\
ldxa [%sp + V9BIAS64 + 96]%asi, %i4 ;\
ldxa [%sp + V9BIAS64 + 104]%asi, %i5 ;\
ldxa [%sp + V9BIAS64 + 112]%asi, %i6 ;\
ldxa [%sp + V9BIAS64 + 120]%asi, %i7 ;\
restored ;\
retry ;\
SKIP(31-19-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_64bit_##tail ;\
.empty
#define SPILL_mixed \
btst 1, %sp ;\
bz,a,pt %xcc, 1b ;\
srl %sp, 0, %sp ;\
ba,pt %xcc, 2b ;\
nop ;\
.align 128
#define FILL_mixed \
btst 1, %sp ;\
bz,a,pt %xcc, 1b ;\
srl %sp, 0, %sp ;\
ba,pt %xcc, 2b ;\
nop ;\
.align 128
#define SPILL_32clean(asi_num, tail) \
srl %sp, 0, %sp ;\
sta %l0, [%sp + %g0]asi_num ;\
mov 4, %g1 ;\
sta %l1, [%sp + %g1]asi_num ;\
mov 8, %g2 ;\
sta %l2, [%sp + %g2]asi_num ;\
mov 12, %g3 ;\
sta %l3, [%sp + %g3]asi_num ;\
add %sp, 16, %g4 ;\
sta %l4, [%g4 + %g0]asi_num ;\
sta %l5, [%g4 + %g1]asi_num ;\
sta %l6, [%g4 + %g2]asi_num ;\
sta %l7, [%g4 + %g3]asi_num ;\
add %g4, 16, %g4 ;\
sta %i0, [%g4 + %g0]asi_num ;\
sta %i1, [%g4 + %g1]asi_num ;\
sta %i2, [%g4 + %g2]asi_num ;\
sta %i3, [%g4 + %g3]asi_num ;\
add %g4, 16, %g4 ;\
sta %i4, [%g4 + %g0]asi_num ;\
sta %i5, [%g4 + %g1]asi_num ;\
sta %i6, [%g4 + %g2]asi_num ;\
sta %i7, [%g4 + %g3]asi_num ;\
TT_TRACE_L(trace_win) ;\
b .spill_clean ;\
mov WSTATE_USER32, %g7 ;\
SKIP(31-25-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_32bit_##tail ;\
.empty
#define SPILL_64clean(asi_num, tail) \
mov 0 + V9BIAS64, %g1 ;\
stxa %l0, [%sp + %g1]asi_num ;\
mov 8 + V9BIAS64, %g2 ;\
stxa %l1, [%sp + %g2]asi_num ;\
mov 16 + V9BIAS64, %g3 ;\
stxa %l2, [%sp + %g3]asi_num ;\
mov 24 + V9BIAS64, %g4 ;\
stxa %l3, [%sp + %g4]asi_num ;\
add %sp, 32, %g5 ;\
stxa %l4, [%g5 + %g1]asi_num ;\
stxa %l5, [%g5 + %g2]asi_num ;\
stxa %l6, [%g5 + %g3]asi_num ;\
stxa %l7, [%g5 + %g4]asi_num ;\
add %g5, 32, %g5 ;\
stxa %i0, [%g5 + %g1]asi_num ;\
stxa %i1, [%g5 + %g2]asi_num ;\
stxa %i2, [%g5 + %g3]asi_num ;\
stxa %i3, [%g5 + %g4]asi_num ;\
add %g5, 32, %g5 ;\
stxa %i4, [%g5 + %g1]asi_num ;\
stxa %i5, [%g5 + %g2]asi_num ;\
stxa %i6, [%g5 + %g3]asi_num ;\
stxa %i7, [%g5 + %g4]asi_num ;\
TT_TRACE_L(trace_win) ;\
b .spill_clean ;\
mov WSTATE_USER64, %g7 ;\
SKIP(31-25-TT_TRACE_L_INS) ;\
ba,a,pt %xcc, fault_64bit_##tail ;\
.empty
#define FP_DISABLED_TRAP \
TT_TRACE(trace_gen) ;\
ba,pt %xcc,.fp_disabled ;\
nop ;\
.align 32
#define FP_IEEE_TRAP \
TT_TRACE(trace_gen) ;\
ba,pt %xcc,.fp_ieee_exception ;\
nop ;\
.align 32
#define FP_TRAP \
TT_TRACE(trace_gen) ;\
ba,pt %xcc,.fp_exception ;\
nop ;\
.align 32
#define ASYNC_TRAP(ttype, ttlabel, table_name)\
.global table_name ;\
table_name: ;\
membar #Sync ;\
TT_TRACE(ttlabel) ;\
ba async_err ;\
mov ttype, %g5 ;\
.align 32
#define LABELED_BAD(table_name) \
.global table_name ;\
table_name: ;\
BAD
#define ILLTRAP_INSTR \
membar #Sync ;\
TT_TRACE(trace_gen) ;\
or %g0, P_UTRAP4, %g2 ;\
or %g0, T_UNIMP_INSTR, %g3 ;\
sethi %hi(.check_v9utrap), %g4 ;\
jmp %g4 + %lo(.check_v9utrap) ;\
nop ;\
.align 32
#define TAG_OVERFLOW \
TT_TRACE(trace_gen) ;\
or %g0, P_UTRAP10, %g2 ;\
or %g0, T_TAG_OVERFLOW, %g3 ;\
sethi %hi(.check_v9utrap), %g4 ;\
jmp %g4 + %lo(.check_v9utrap) ;\
nop ;\
.align 32
#define DIV_BY_ZERO \
TT_TRACE(trace_gen) ;\
or %g0, P_UTRAP11, %g2 ;\
or %g0, T_IDIV0, %g3 ;\
sethi %hi(.check_v9utrap), %g4 ;\
jmp %g4 + %lo(.check_v9utrap) ;\
nop ;\
.align 32
#define TRAP_INSTR \
TT_TRACE(trace_gen) ;\
or %g0, T_SOFTWARE_TRAP, %g3 ;\
sethi %hi(.check_v9utrap), %g4 ;\
jmp %g4 + %lo(.check_v9utrap) ;\
nop ;\
.align 32
#define TRP4 TRAP_INSTR; TRAP_INSTR; TRAP_INSTR; TRAP_INSTR
#define LEVEL_INTERRUPT(level) \
.global tt_pil##level ;\
tt_pil##level: ;\
ba,pt %xcc, pil_interrupt ;\
mov level, %g4 ;\
.align 32
#define LEVEL14_INTERRUPT \
ba pil14_interrupt ;\
mov PIL_14, %g4 ;\
.align 32
#define LEVEL15_INTERRUPT \
ba pil15_interrupt ;\
mov PIL_15, %g4 ;\
.align 32
#define VECTOR_INTERRUPT \
ldxa [%g0]ASI_INTR_RECEIVE_STATUS, %g1 ;\
btst IRSR_BUSY, %g1 ;\
bnz,pt %xcc, vec_interrupt ;\
nop ;\
ba,a,pt %xcc, vec_intr_spurious ;\
.empty ;\
.align 32
#define SWITCH_GLOBALS \
rdpr %pstate, %g5 ;\
wrpr %g5, PSTATE_MG | PSTATE_AG, %pstate
#define IMMU_EXCEPTION \
membar #Sync ;\
SWITCH_GLOBALS ;\
wr %g0, ASI_IMMU, %asi ;\
rdpr %tpc, %g2 ;\
ldxa [MMU_SFSR]%asi, %g3 ;\
ba,pt %xcc, .mmu_exception_end ;\
mov T_INSTR_EXCEPTION, %g1 ;\
.align 32
#define DMMU_EXCEPTION \
SWITCH_GLOBALS ;\
wr %g0, ASI_DMMU, %asi ;\
ldxa [MMU_TAG_ACCESS]%asi, %g2 ;\
ldxa [MMU_SFSR]%asi, %g3 ;\
ba,pt %xcc, .mmu_exception_end ;\
mov T_DATA_EXCEPTION, %g1 ;\
.align 32
#define DMMU_EXC_AG_PRIV \
wr %g0, ASI_DMMU, %asi ;\
ldxa [MMU_SFAR]%asi, %g2 ;\
ba,pt %xcc, .mmu_priv_exception ;\
ldxa [MMU_SFSR]%asi, %g3 ;\
.align 32
#define DMMU_EXC_AG_NOT_ALIGNED \
wr %g0, ASI_DMMU, %asi ;\
ldxa [MMU_SFAR]%asi, %g2 ;\
ba,pt %xcc, .mmu_exception_not_aligned ;\
ldxa [MMU_SFSR]%asi, %g3 ;\
.align 32
#define DMMU_EXC_LDDF_NOT_ALIGNED \
btst 1, %sp ;\
bnz,pt %xcc, .lddf_exception_not_aligned ;\
wr %g0, ASI_DMMU, %asi ;\
ldxa [MMU_SFAR]%asi, %g2 ;\
ba,pt %xcc, .mmu_exception_not_aligned ;\
ldxa [MMU_SFSR]%asi, %g3 ;\
.align 32
#define DMMU_EXC_STDF_NOT_ALIGNED \
btst 1, %sp ;\
bnz,pt %xcc, .stdf_exception_not_aligned ;\
wr %g0, ASI_DMMU, %asi ;\
ldxa [MMU_SFAR]%asi, %g2 ;\
ba,pt %xcc, .mmu_exception_not_aligned ;\
ldxa [MMU_SFSR]%asi, %g3 ;\
.align 32
#if TAGACC_CTX_MASK != CTXREG_CTX_MASK
#error "TAGACC_CTX_MASK != CTXREG_CTX_MASK"
#endif
#define DTLB_DEMAP_ENTRY \
mov MMU_TAG_ACCESS, %g1 ;\
mov MMU_PCONTEXT, %g5 ;\
ldxa [%g1]ASI_DMMU, %g2 ;\
sethi %hi(TAGACC_CTX_MASK), %g4 ;\
or %g4, %lo(TAGACC_CTX_MASK), %g4 ;\
and %g2, %g4, %g3 ;\
ldxa [%g5]ASI_DMMU, %g6 ;\
and %g6, %g4, %g6 ;\
cmp %g3, %g6 ;\
be,pt %xcc, 1f ;\
andn %g2, %g4, %g1 ;\
mov MMU_SCONTEXT, %g5 ;\
ldxa [%g5]ASI_DMMU, %g6 ;\
and %g6, %g4, %g6 ;\
cmp %g3, %g6 ;\
be,a,pt %xcc, 1f ;\
or %g1, DEMAP_SECOND, %g1 ;\
or %g1, DEMAP_NUCLEUS, %g1 ;\
1: stxa %g0, [%g1]ASI_DTLB_DEMAP ;\
membar #Sync
#if defined(cscope)
.global tt0_dtlbmiss
tt0_dtlbmiss:
.global tt1_dtlbmiss
tt1_dtlbmiss:
nop
#endif
#ifdef UTSB_PHYS
#define DTLB_MISS(table_name) ;\
.global table_name##_dtlbmiss ;\
table_name##_dtlbmiss: ;\
mov MMU_TAG_ACCESS, %g6 ;\
ldxa [%g0]ASI_DMMU_TSB_8K, %g1 ;\
ldxa [%g6]ASI_DMMU, %g2 ;\
sllx %g2, TAGACC_CTX_LSHIFT, %g3 ;\
srlx %g3, TAGACC_CTX_LSHIFT, %g3 ;\
cmp %g3, INVALID_CONTEXT ;\
ble,pn %xcc, sfmmu_kdtlb_miss ;\
srax %g2, PREDISM_BASESHIFT, %g6 ;\
brgz,pn %g6, sfmmu_udtlb_slowpath_ismpred ;\
srlx %g2, TAG_VALO_SHIFT, %g7 ;\
ldda [%g1]ASI_QUAD_LDD_PHYS, %g4 ;\
cmp %g4, %g7 ;\
bne,pn %xcc, sfmmu_udtlb_slowpath_noismpred ;\
nop ;\
TT_TRACE(trace_tsbhit) ;\
stxa %g5, [%g0]ASI_DTLB_IN ;\
retry ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
.align 128
#else
#define DTLB_MISS(table_name) ;\
.global table_name##_dtlbmiss ;\
table_name##_dtlbmiss: ;\
mov MMU_TAG_ACCESS, %g6 ;\
ldxa [%g0]ASI_DMMU_TSB_8K, %g1 ;\
ldxa [%g6]ASI_DMMU, %g2 ;\
sllx %g2, TAGACC_CTX_LSHIFT, %g3 ;\
srlx %g3, TAGACC_CTX_LSHIFT, %g3 ;\
cmp %g3, INVALID_CONTEXT ;\
ble,pn %xcc, sfmmu_kdtlb_miss ;\
srlx %g2, TAG_VALO_SHIFT, %g7 ;\
brlz,pn %g1, sfmmu_udtlb_slowpath ;\
nop ;\
ldda [%g1]ASI_NQUAD_LD, %g4 ;\
cmp %g4, %g7 ;\
bne,pn %xcc, sfmmu_tsb_miss_tt ;\
mov -1, %g3 ;\
TT_TRACE(trace_tsbhit) ;\
stxa %g5, [%g0]ASI_DTLB_IN ;\
retry ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
.align 128
#endif
#if defined(cscope)
.global tt0_itlbmiss
tt0_itlbmiss:
.global tt1_itlbmiss
tt1_itlbmiss:
nop
#endif
#ifdef UTSB_PHYS
#define ITLB_MISS(table_name) \
.global table_name##_itlbmiss ;\
table_name##_itlbmiss: ;\
mov MMU_TAG_ACCESS, %g6 ;\
ldxa [%g0]ASI_IMMU_TSB_8K, %g1 ;\
ldxa [%g6]ASI_IMMU, %g2 ;\
sllx %g2, TAGACC_CTX_LSHIFT, %g3 ;\
srlx %g3, TAGACC_CTX_LSHIFT, %g3 ;\
cmp %g3, INVALID_CONTEXT ;\
ble,pn %xcc, sfmmu_kitlb_miss ;\
srlx %g2, TAG_VALO_SHIFT, %g7 ;\
ldda [%g1]ASI_QUAD_LDD_PHYS, %g4 ;\
cmp %g4, %g7 ;\
bne,pn %xcc, sfmmu_uitlb_slowpath ;\
andcc %g5, TTE_EXECPRM_INT, %g0 ;\
bz,pn %icc, exec_fault ;\
nop ;\
TT_TRACE(trace_tsbhit) ;\
stxa %g5, [%g0]ASI_ITLB_IN ;\
retry ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
.align 128
#else
#define ITLB_MISS(table_name) \
.global table_name##_itlbmiss ;\
table_name##_itlbmiss: ;\
mov MMU_TAG_ACCESS, %g6 ;\
ldxa [%g0]ASI_IMMU_TSB_8K, %g1 ;\
ldxa [%g6]ASI_IMMU, %g2 ;\
sllx %g2, TAGACC_CTX_LSHIFT, %g3 ;\
srlx %g3, TAGACC_CTX_LSHIFT, %g3 ;\
cmp %g3, INVALID_CONTEXT ;\
ble,pn %xcc, sfmmu_kitlb_miss ;\
srlx %g2, TAG_VALO_SHIFT, %g7 ;\
brlz,pn %g1, sfmmu_uitlb_slowpath ;\
nop ;\
ldda [%g1]ASI_NQUAD_LD, %g4 ;\
cmp %g4, %g7 ;\
bne,pn %xcc, sfmmu_tsb_miss_tt ;\
mov -1, %g3 ;\
andcc %g5, TTE_EXECPRM_INT, %g0 ;\
bz,pn %icc, exec_fault ;\
nop ;\
TT_TRACE(trace_tsbhit) ;\
stxa %g5, [%g0]ASI_ITLB_IN ;\
retry ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
.align 128
#endif
#define DTLB_PROT \
DTLB_DEMAP_ENTRY ;\
;\
TT_TRACE(trace_dataprot) ;\
;\
ldxa [%g0]ASI_DMMU_TSB_8K, %g1 ;\
brnz,pt %g3, sfmmu_uprot_trap ;\
nop ;\
ba,a,pt %xcc, sfmmu_kprot_trap ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
unimp 0 ;\
.align 128
#define DMMU_EXCEPTION_TL1 ;\
SWITCH_GLOBALS ;\
ba,a,pt %xcc, mmu_trap_tl1 ;\
nop ;\
.align 32
#define MISALIGN_ADDR_TL1 ;\
ba,a,pt %xcc, mmu_trap_tl1 ;\
nop ;\
.align 32
#ifdef TRAPTRACE
#define TRACE_TSBHIT(ttextra) \
membar #Sync ;\
sethi %hi(FLUSH_ADDR), %g6 ;\
flush %g6 ;\
TRACE_PTR(%g3, %g6) ;\
GET_TRACE_TICK(%g6, %g4) ;\
stxa %g6, [%g3 + TRAP_ENT_TICK]%asi ;\
stxa %g2, [%g3 + TRAP_ENT_SP]%asi ;\
stxa %g5, [%g3 + TRAP_ENT_F1]%asi ;\
rdpr %tnpc, %g6 ;\
stxa %g6, [%g3 + TRAP_ENT_F2]%asi ;\
stxa %g1, [%g3 + TRAP_ENT_F3]%asi ;\
stxa %g0, [%g3 + TRAP_ENT_F4]%asi ;\
rdpr %tpc, %g6 ;\
stxa %g6, [%g3 + TRAP_ENT_TPC]%asi ;\
rdpr %tl, %g6 ;\
stha %g6, [%g3 + TRAP_ENT_TL]%asi ;\
rdpr %tt, %g6 ;\
or %g6, (ttextra), %g6 ;\
stha %g6, [%g3 + TRAP_ENT_TT]%asi ;\
ldxa [%g0]ASI_IMMU, %g1 ;\
ldxa [%g0]ASI_DMMU, %g4 ;\
cmp %g6, FAST_IMMU_MISS_TT ;\
movne %icc, %g4, %g1 ;\
stxa %g1, [%g3 + TRAP_ENT_TSTATE]%asi ;\
stxa %g0, [%g3 + TRAP_ENT_TR]%asi ;\
TRACE_NEXT(%g3, %g4, %g6)
#else
#define TRACE_TSBHIT(ttextra)
#endif
.section ".text"
.align 4
.global trap_table, scb, trap_table0, trap_table1, etrap_table
.type trap_table, #object
.type scb, #object
trap_table:
scb:
trap_table0:
NOT;
RED;
RED;
RED;
RED;
RED;
NOT; NOT;
IMMU_EXCEPTION;
NOT;
ASYNC_TRAP(T_INSTR_ERROR, trace_gen, tt0_iae);
NOT; NOT4;
ILLTRAP_INSTR;
TRAP(T_PRIV_INSTR);
NOT;
NOT;
NOT4; NOT4; NOT4;
FP_DISABLED_TRAP;
FP_IEEE_TRAP;
FP_TRAP;
TAG_OVERFLOW;
CLEAN_WINDOW;
DIV_BY_ZERO;
NOT;
NOT; NOT; NOT4;
DMMU_EXCEPTION;
NOT;
ASYNC_TRAP(T_DATA_ERROR, trace_gen, tt0_dae);
NOT;
DMMU_EXC_AG_NOT_ALIGNED;
DMMU_EXC_LDDF_NOT_ALIGNED;
DMMU_EXC_STDF_NOT_ALIGNED;
DMMU_EXC_AG_PRIV;
NOT;
NOT;
NOT; NOT; NOT4;
LABELED_BAD(tt0_asdat);
LEVEL_INTERRUPT(1);
LEVEL_INTERRUPT(2);
LEVEL_INTERRUPT(3);
LEVEL_INTERRUPT(4);
LEVEL_INTERRUPT(5);
LEVEL_INTERRUPT(6);
LEVEL_INTERRUPT(7);
LEVEL_INTERRUPT(8);
LEVEL_INTERRUPT(9);
LEVEL_INTERRUPT(10);
LEVEL_INTERRUPT(11);
LEVEL_INTERRUPT(12);
LEVEL_INTERRUPT(13);
LEVEL14_INTERRUPT;
LEVEL15_INTERRUPT;
NOT4; NOT4; NOT4; NOT4;
VECTOR_INTERRUPT;
GOTO(kmdb_trap);
GOTO(kmdb_trap);
GOTO_TT(ce_err, trace_gen);
ITLB_MISS(tt0);
DTLB_MISS(tt0);
DTLB_PROT;
LABELED_BAD(tt0_fecc);
LABELED_BAD(tt0_dperr);
LABELED_BAD(tt0_iperr);
NOT;
NOT4; NOT4; NOT4;
NOT4;
SPILL_32bit_asi(ASI_AIUP,sn0);
SPILL_64bit_asi(ASI_AIUP,sn0);
SPILL_32clean(ASI_AIUP,sn0);
SPILL_64clean(ASI_AIUP,sn0);
SPILL_32bit(not);
SPILL_64bit(not);
SPILL_mixed;
NOT4;
SPILL_32bit_asi(ASI_AIUS,so0);
SPILL_64bit_asi(ASI_AIUS,so0);
SPILL_32bit_asi(ASI_AIUS,so0);
SPILL_64bit_asi(ASI_AIUS,so0);
NOT4;
NOT4;
NOT4;
NOT4;
FILL_32bit_asi(ASI_AIUP,fn0);
FILL_64bit_asi(ASI_AIUP,fn0);
FILL_32bit_asi(ASI_AIUP,fn0);
FILL_64bit_asi(ASI_AIUP,fn0);
FILL_32bit(not);
FILL_64bit(not);
FILL_mixed;
NOT4;
NOT4;
NOT4;
NOT4;
NOT4;
NOT4;
NOT4;
NOT4;
GOTO(syscall_trap_4x);
TRAP(T_BREAKPOINT);
TRAP(T_DIV0);
FLUSHW(tt0_flushw);
GOTO(.clean_windows);
BAD;
GOTO(.fix_alignment);
BAD;
SYSCALL_TRAP32;
GOTO(set_trap0_addr);
BAD; BAD; BAD4;
TRP4; TRP4; TRP4; TRP4;
GOTO(.getcc);
GOTO(.setcc);
GOTO(.getpsr);
GOTO(.setpsr);
GOTO(get_timestamp);
GOTO(get_virtime);
PRIV(self_xcall);
GOTO(get_hrestime);
BAD;
GOTO(.getlgrp);
BAD; BAD; BAD4;
BAD4; BAD4;
DTRACE_PID;
BAD;
DTRACE_RETURN;
BAD; BAD4;
SYSCALL_TRAP;
SYSCALL(nosys);
#ifdef DEBUG_USER_TRAPTRACECTL
GOTO(.traptrace_freeze);
GOTO(.traptrace_unfreeze);
#else
SYSCALL(nosys);
SYSCALL(nosys);
#endif
BAD4; BAD4; BAD4;
BAD4; BAD4; BAD4; BAD4;
BAD4; BAD4; BAD4; BAD4;
BAD;
BAD;
BAD; BAD;
BAD4; BAD4;
#ifdef PTL1_PANIC_DEBUG
mov PTL1_BAD_DEBUG, %g1; GOTO(ptl1_panic);
#else
BAD;
#endif
PRIV(kmdb_trap);
PRIV(kmdb_trap);
PRIV(kctx_obp_bpt);
NOT4; NOT4; NOT4; NOT4;
NOT4; NOT4; NOT4; NOT4;
NOT4; NOT4; NOT4; NOT4;
NOT4; NOT4; NOT4; NOT4;
NOT4; NOT4; NOT4; NOT4;
NOT4; NOT4; NOT4; NOT4;
NOT4; NOT4; NOT4; NOT4;
NOT4; NOT4; NOT4; NOT4;
trap_table1:
NOT4; NOT4; NOT; NOT;
ASYNC_TRAP(T_INSTR_ERROR + T_TL1, trace_gen, tt1_iae);
NOT; NOT4;
NOT4; NOT4; NOT4; NOT4;
NOT4;
CLEAN_WINDOW;
NOT4; NOT4;
DMMU_EXCEPTION_TL1;
NOT;
ASYNC_TRAP(T_DATA_ERROR + T_TL1, trace_gen, tt1_dae);
NOT;
MISALIGN_ADDR_TL1;
NOT; NOT; NOT; NOT4; NOT4
LABELED_BAD(tt1_asdat);
NOT; NOT; NOT;
NOT4; NOT4; NOT4;
NOT4; NOT4; NOT4; NOT4;
NOT;
GOTO(kmdb_trap_tl1);
GOTO(kmdb_trap_tl1);
GOTO_TT(ce_err_tl1, trace_gen);
ITLB_MISS(tt1);
DTLB_MISS(tt1);
DTLB_PROT;
LABELED_BAD(tt1_fecc);
LABELED_BAD(tt1_dperr);
LABELED_BAD(tt1_iperr);
NOT;
NOT4; NOT4; NOT4;
NOT4;
SPILL_32bit_tt1(ASI_AIUP,sn1);
SPILL_64bit_tt1(ASI_AIUP,sn1);
SPILL_32bit_tt1(ASI_AIUP,sn1);
SPILL_64bit_tt1(ASI_AIUP,sn1);
SPILL_32bit(not);
SPILL_64bit(not);
SPILL_mixed;
NOT4;
SPILL_32bit_tt1(ASI_AIUS,so1);
SPILL_64bit_tt1(ASI_AIUS,so1);
SPILL_32bit_tt1(ASI_AIUS,so1);
SPILL_64bit_tt1(ASI_AIUS,so1);
NOT4;
NOT4;
NOT4;
NOT4;
FILL_32bit_tt1(ASI_AIUP,fn1);
FILL_64bit_tt1(ASI_AIUP,fn1);
FILL_32bit_tt1(ASI_AIUP,fn1);
FILL_64bit_tt1(ASI_AIUP,fn1);
FILL_32bit(not);
FILL_64bit(not);
FILL_mixed;
NOT4; NOT4; NOT4; NOT4;
NOT4; NOT4; NOT4; NOT4;
LABELED_BAD(tt1_swtrap0);
LABELED_BAD(tt1_swtrap1);
LABELED_BAD(tt1_swtrap2);
NOT;
etrap_table:
.size trap_table, (.-trap_table)
.size scb, (.-scb)
ALTENTRY(exec_fault)
TRACE_TSBHIT(0x200)
SWITCH_GLOBALS
mov MMU_TAG_ACCESS, %g4
ldxa [%g4]ASI_IMMU, %g2 ! arg1 = addr
mov T_INSTR_MMU_MISS, %g3 ! arg2 = traptype
set trap, %g1
ba,pt %xcc, sys_trap
mov -1, %g4
.mmu_exception_not_aligned:
rdpr %tstate, %g1
btst TSTATE_PRIV, %g1
bnz,pn %icc, 2f
nop
CPU_ADDR(%g1, %g4) ! load CPU struct addr
ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
ldn [%g1 + T_PROCP], %g1 ! load proc pointer
ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
brz,pt %g5, 2f
nop
ldn [%g5 + P_UTRAP15], %g5 ! unaligned utrap?
brz,pn %g5, 2f
nop
btst 1, %sp
bz,pt %xcc, 1f ! 32 bit user program
nop
ba,pt %xcc, .setup_v9utrap ! 64 bit user program
nop
1:
ba,pt %xcc, .setup_utrap
or %g2, %g0, %g7
2:
ba,pt %xcc, .mmu_exception_end
mov T_ALIGNMENT, %g1
.mmu_priv_exception:
rdpr %tstate, %g1
btst TSTATE_PRIV, %g1
bnz,pn %icc, 1f
nop
CPU_ADDR(%g1, %g4) ! load CPU struct addr
ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
ldn [%g1 + T_PROCP], %g1 ! load proc pointer
ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
brz,pt %g5, 1f
nop
ldn [%g5 + P_UTRAP16], %g5
brnz,pt %g5, .setup_v9utrap
nop
1:
mov T_PRIV_INSTR, %g1
.mmu_exception_end:
CPU_INDEX(%g4, %g5)
set cpu_core, %g5
sllx %g4, CPU_CORE_SHIFT, %g4
add %g4, %g5, %g4
lduh [%g4 + CPUC_DTRACE_FLAGS], %g5
andcc %g5, CPU_DTRACE_NOFAULT, %g0
bz %xcc, .mmu_exception_tlb_chk
or %g5, CPU_DTRACE_BADADDR, %g5
stuh %g5, [%g4 + CPUC_DTRACE_FLAGS]
done
.mmu_exception_tlb_chk:
GET_CPU_IMPL(%g5) ! check SFSR.FT to see if this
cmp %g5, PANTHER_IMPL ! is a TLB parity error. But
bne 2f ! we only do this check while
mov 1, %g4 ! running on Panther CPUs
sllx %g4, PN_SFSR_PARITY_SHIFT, %g4 ! since US-I/II use the same
andcc %g3, %g4, %g0 ! bit for something else which
bz 2f ! will be handled later.
nop
.mmu_exception_is_tlb_parity:
.weak itlb_parity_trap
.weak dtlb_parity_trap
set itlb_parity_trap, %g4
cmp %g1, T_INSTR_EXCEPTION ! branch to the itlb or
be 3f ! dtlb parity handler
nop ! if this trap is due
set dtlb_parity_trap, %g4
cmp %g1, T_DATA_EXCEPTION ! to a IMMU exception
be 3f ! or DMMU exception.
nop
2:
sllx %g3, 32, %g3
or %g3, %g1, %g3
set trap, %g1
ba,pt %xcc, sys_trap
sub %g0, 1, %g4
3:
jmp %g4 ! off to the appropriate
nop ! TLB parity handler
.fp_disabled:
CPU_ADDR(%g1, %g4) ! load CPU struct addr
ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
#ifdef SF_ERRATA_30
brz,a,pn %g1, 2f
nop
#endif
rdpr %tstate, %g4
btst TSTATE_PRIV, %g4
#ifdef SF_ERRATA_30
bnz,pn %icc, 2f
nop
#else
bnz,a,pn %icc, ptl1_panic
mov PTL1_BAD_FPTRAP, %g1
#endif
ldn [%g1 + T_PROCP], %g1 ! load proc pointer
ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
brz,a,pt %g5, 2f
nop
ldn [%g5 + P_UTRAP7], %g5 ! fp_disabled utrap?
brz,a,pn %g5, 2f
nop
btst 1, %sp
bz,a,pt %xcc, 1f ! 32 bit user program
nop
ba,a,pt %xcc, .setup_v9utrap ! 64 bit user program
nop
1:
ba,pt %xcc, .setup_utrap
or %g0, %g0, %g7
2:
set fp_disabled, %g1
ba,pt %xcc, sys_trap
sub %g0, 1, %g4
.fp_ieee_exception:
rdpr %tstate, %g1
btst TSTATE_PRIV, %g1
bnz,a,pn %icc, ptl1_panic
mov PTL1_BAD_FPTRAP, %g1
CPU_ADDR(%g1, %g4) ! load CPU struct addr
stx %fsr, [%g1 + CPU_TMP1]
ldx [%g1 + CPU_TMP1], %g2
ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
ldn [%g1 + T_PROCP], %g1 ! load proc pointer
ldn [%g1 + P_UTRAPS], %g5 ! are there utraps?
brz,a,pt %g5, 1f
nop
ldn [%g5 + P_UTRAP8], %g5
brnz,a,pt %g5, .setup_v9utrap
nop
1:
set _fp_ieee_exception, %g1
ba,pt %xcc, sys_trap
sub %g0, 1, %g4
.setup_utrap:
set trap, %g1 ! setup in case we go
mov T_FLUSH_PCB, %g3 ! through sys_trap on
sub %g0, 1, %g4 ! the save instruction below
save %sp, -SA(MINFRAME32), %sp ! window for trap handler
CPU_ADDR(%g1, %g4) ! load CPU struct addr
ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
ldub [%g1 + T_DTRACE_STEP], %g2 ! load t->t_dtrace_step
rdpr %tnpc, %l2 ! arg1 == tnpc
brz,pt %g2, 1f
rdpr %tpc, %l1 ! arg0 == tpc
ldub [%g1 + T_DTRACE_AST], %g2 ! load t->t_dtrace_ast
ldn [%g1 + T_DTRACE_NPC], %l2 ! arg1 = t->t_dtrace_npc (step)
brz,pt %g2, 1f
st %g0, [%g1 + T_DTRACE_FT] ! zero all pid provider flags
stub %g2, [%g1 + T_ASTFLAG] ! aston(t) if t->t_dtrace_ast
1:
mov %g7, %l3 ! arg2 == misaligned address
rdpr %tstate, %g1 ! cwp for trap handler
rdpr %cwp, %g4
bclr TSTATE_CWP_MASK, %g1
wrpr %g1, %g4, %tstate
wrpr %g0, %g5, %tnpc ! trap handler address
FAST_TRAP_DONE
.check_v9utrap:
rdpr %tstate, %g1
btst TSTATE_PRIV, %g1
bnz,a,pn %icc, 3f
nop
CPU_ADDR(%g4, %g1) ! load CPU struct addr
ldn [%g4 + CPU_THREAD], %g5 ! load thread pointer
ldn [%g5 + T_PROCP], %g5 ! load proc pointer
ldn [%g5 + P_UTRAPS], %g5 ! are there utraps?
cmp %g3, T_SOFTWARE_TRAP
bne,a,pt %icc, 1f
nop
brz,pt %g5, 3f ! if p_utraps == NULL goto trap()
rdpr %tt, %g3 ! delay - get actual hw trap type
sub %g3, 254, %g1 ! UT_TRAP_INSTRUCTION_16 = p_utraps[18]
ba,pt %icc, 2f
smul %g1, CPTRSIZE, %g2
1:
brz,a,pt %g5, 3f ! if p_utraps == NULL goto trap()
nop
cmp %g3, T_UNIMP_INSTR
bne,a,pt %icc, 2f
nop
mov 1, %g1
st %g1, [%g4 + CPU_TL1_HDLR] ! set CPU_TL1_HDLR
rdpr %tpc, %g1 ! ld trapping instruction using
lduwa [%g1]ASI_AIUP, %g1 ! "AS IF USER" ASI which could fault
st %g0, [%g4 + CPU_TL1_HDLR] ! clr CPU_TL1_HDLR
sethi %hi(0xc1c00000), %g4 ! setup mask for illtrap instruction
andcc %g1, %g4, %g4 ! and instruction with mask
bnz,a,pt %icc, 3f ! if %g4 == zero, %g1 is an ILLTRAP
nop ! fall thru to setup
2:
ldn [%g5 + %g2], %g5
brnz,a,pt %g5, .setup_v9utrap
nop
3:
set trap, %g1
ba,pt %xcc, sys_trap
sub %g0, 1, %g4
.setup_v9utrap:
set trap, %g1 ! setup in case we go
mov T_FLUSH_PCB, %g3 ! through sys_trap on
sub %g0, 1, %g4 ! the save instruction below
save %sp, -SA(MINFRAME64), %sp ! window for trap handler
CPU_ADDR(%g1, %g4) ! load CPU struct addr
ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
ldub [%g1 + T_DTRACE_STEP], %g2 ! load t->t_dtrace_step
rdpr %tnpc, %l7 ! arg1 == tnpc
brz,pt %g2, 1f
rdpr %tpc, %l6 ! arg0 == tpc
ldub [%g1 + T_DTRACE_AST], %g2 ! load t->t_dtrace_ast
ldn [%g1 + T_DTRACE_NPC], %l7 ! arg1 == t->t_dtrace_npc (step)
brz,pt %g2, 1f
st %g0, [%g1 + T_DTRACE_FT] ! zero all pid provider flags
stub %g2, [%g1 + T_ASTFLAG] ! aston(t) if t->t_dtrace_ast
1:
rdpr %tstate, %g2 ! cwp for trap handler
rdpr %cwp, %g4
bclr TSTATE_CWP_MASK, %g2
wrpr %g2, %g4, %tstate
ldn [%g1 + T_PROCP], %g4 ! load proc pointer
ldn [%g4 + P_AS], %g4 ! load as pointer
ldn [%g4 + A_USERLIMIT], %g4 ! load as userlimit
cmp %l7, %g4 ! check for single-step set
bne,pt %xcc, 4f
nop
ldn [%g1 + T_LWP], %g1 ! load klwp pointer
ld [%g1 + PCB_STEP], %g4 ! load single-step flag
cmp %g4, STEP_ACTIVE ! step flags set in pcb?
bne,pt %icc, 4f
nop
stn %g5, [%g1 + PCB_TRACEPC] ! save trap handler addr in pcb
mov %l7, %g4 ! on entry to precise user trap
add %l6, 4, %l7 ! handler, %l6 == pc, %l7 == npc
! at time of trap
wrpr %g0, %g4, %tnpc ! generate FLTBOUNDS,
! %g4 == userlimit
FAST_TRAP_DONE
4:
wrpr %g0, %g5, %tnpc ! trap handler address
FAST_TRAP_DONE_CHK_INTR
.fp_exception:
CPU_ADDR(%g1, %g4)
stx %fsr, [%g1 + CPU_TMP1]
ldx [%g1 + CPU_TMP1], %g2
#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
sethi %hi(fpu_exists), %g7
ld [%g7 + %lo(fpu_exists)], %g7
brz,pn %g7, .fp_exception_cont
nop
#endif
rdpr %tstate, %g7 ! branch if in privileged mode
btst TSTATE_PRIV, %g7
bnz,pn %xcc, .fp_exception_cont
srl %g2, FSR_FTT_SHIFT, %g7 ! extract ftt from %fsr
and %g7, (FSR_FTT>>FSR_FTT_SHIFT), %g7
cmp %g7, FTT_UNFIN
set FSR_TEM_NX, %g5
bne,pn %xcc, .fp_exception_cont ! branch if NOT unfinished_FPop
andcc %g2, %g5, %g0
bne,pn %xcc, .fp_exception_cont ! branch if FSR_TEM_NX enabled
rdpr %tpc, %g5 ! get faulting PC
or %g0, 1, %g7
st %g7, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
lda [%g5]ASI_USER, %g6 ! get user's instruction
st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
set FITOS_INSTR_MASK, %g7
and %g6, %g7, %g7
set FITOS_INSTR, %g5
cmp %g7, %g5
bne,pn %xcc, .fp_exception_cont ! branch if not FITOS_INSTR
nop
std %d62, [%g1 + CPU_TMP1] ! save original value
srl %g6, FITOS_RS2_SHIFT, %g7
and %g7, FITOS_REG_MASK, %g7
set _fitos_fitod_table, %g4
sllx %g7, 2, %g7
jmp %g4 + %g7
ba,pt %xcc, _fitos_fitod_done
.empty
_fitos_fitod_table:
fitod %f0, %d62
fitod %f1, %d62
fitod %f2, %d62
fitod %f3, %d62
fitod %f4, %d62
fitod %f5, %d62
fitod %f6, %d62
fitod %f7, %d62
fitod %f8, %d62
fitod %f9, %d62
fitod %f10, %d62
fitod %f11, %d62
fitod %f12, %d62
fitod %f13, %d62
fitod %f14, %d62
fitod %f15, %d62
fitod %f16, %d62
fitod %f17, %d62
fitod %f18, %d62
fitod %f19, %d62
fitod %f20, %d62
fitod %f21, %d62
fitod %f22, %d62
fitod %f23, %d62
fitod %f24, %d62
fitod %f25, %d62
fitod %f26, %d62
fitod %f27, %d62
fitod %f28, %d62
fitod %f29, %d62
fitod %f30, %d62
fitod %f31, %d62
_fitos_fitod_done:
srl %g6, FITOS_RD_SHIFT, %g7
and %g7, FITOS_REG_MASK, %g7
set _fitos_fdtos_table, %g4
sllx %g7, 2, %g7
jmp %g4 + %g7
ba,pt %xcc, _fitos_fdtos_done
.empty
_fitos_fdtos_table:
fdtos %d62, %f0
fdtos %d62, %f1
fdtos %d62, %f2
fdtos %d62, %f3
fdtos %d62, %f4
fdtos %d62, %f5
fdtos %d62, %f6
fdtos %d62, %f7
fdtos %d62, %f8
fdtos %d62, %f9
fdtos %d62, %f10
fdtos %d62, %f11
fdtos %d62, %f12
fdtos %d62, %f13
fdtos %d62, %f14
fdtos %d62, %f15
fdtos %d62, %f16
fdtos %d62, %f17
fdtos %d62, %f18
fdtos %d62, %f19
fdtos %d62, %f20
fdtos %d62, %f21
fdtos %d62, %f22
fdtos %d62, %f23
fdtos %d62, %f24
fdtos %d62, %f25
fdtos %d62, %f26
fdtos %d62, %f27
fdtos %d62, %f28
fdtos %d62, %f29
fdtos %d62, %f30
fdtos %d62, %f31
_fitos_fdtos_done:
ldd [%g1 + CPU_TMP1], %d62 ! restore %d62
#if DEBUG
set fpustat+FPUSTAT_UNFIN_KSTAT, %g7
ldx [%g7], %g5
1:
add %g5, 1, %g6
casxa [%g7] ASI_N, %g5, %g6
cmp %g5, %g6
bne,a,pn %xcc, 1b
or %g0, %g6, %g5
set fpuinfo+FPUINFO_FITOS_KSTAT, %g7
ldx [%g7], %g5
1:
add %g5, 1, %g6
casxa [%g7] ASI_N, %g5, %g6
cmp %g5, %g6
bne,a,pn %xcc, 1b
or %g0, %g6, %g5
#endif
FAST_TRAP_DONE
.fp_exception_cont:
set _fp_exception, %g1
ba,pt %xcc, sys_trap
sub %g0, 1, %g4
.global opl_cleanw_patch
opl_cleanw_patch:
.clean_windows:
set trap, %g1
mov T_FLUSH_PCB, %g3
sub %g0, 1, %g4
save
flushw
restore
wrpr %g0, %g0, %cleanwin ! no clean windows
CPU_ADDR(%g4, %g5)
ldn [%g4 + CPU_MPCB], %g4
brz,a,pn %g4, 1f
nop
ld [%g4 + MPCB_WSTATE], %g5
add %g5, WSTATE_CLEAN_OFFSET, %g5
wrpr %g0, %g5, %wstate
1: FAST_TRAP_DONE
.spill_clean:
sethi %hi(nwin_minus_one), %g5
ld [%g5 + %lo(nwin_minus_one)], %g5 ! %g5 = nwin - 1
rdpr %cwp, %g6 ! %g6 = %cwp
deccc %g6 ! %g6--
movneg %xcc, %g5, %g6 ! if (%g6<0) %g6 = nwin-1
wrpr %g6, %cwp
TT_TRACE_L(trace_win)
clr %l0
clr %l1
clr %l2
clr %l3
clr %l4
clr %l5
clr %l6
clr %l7
wrpr %g0, %g7, %wstate
saved
retry ! restores correct %cwp
.fix_alignment:
CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
ldn [%g1 + CPU_THREAD], %g1 ! load thread pointer
ldn [%g1 + T_PROCP], %g1
mov 1, %g2
stb %g2, [%g1 + P_FIXALIGNMENT]
FAST_TRAP_DONE
#define STDF_REG(REG, ADDR, TMP) \
sll REG, 3, REG ;\
mark1: set start1, TMP ;\
jmp REG + TMP ;\
nop ;\
start1: ba,pt %xcc, done1 ;\
std %f0, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f32, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f2, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f34, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f4, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f36, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f6, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f38, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f8, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f40, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f10, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f42, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f12, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f44, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f14, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f46, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f16, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f48, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f18, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f50, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f20, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f52, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f22, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f54, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f24, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f56, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f26, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f58, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f28, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f60, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f30, [ADDR + CPU_TMP1] ;\
ba,pt %xcc, done1 ;\
std %f62, [ADDR + CPU_TMP1] ;\
done1:
#define LDDF_REG(REG, ADDR, TMP) \
sll REG, 3, REG ;\
mark2: set start2, TMP ;\
jmp REG + TMP ;\
nop ;\
start2: ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f0 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f32 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f2 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f34 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f4 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f36 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f6 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f38 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f8 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f40 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f10 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f42 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f12 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f44 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f14 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f46 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f16 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f48 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f18 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f50 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f20 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f52 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f22 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f54 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f24 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f56 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f26 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f58 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f28 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f60 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f30 ;\
ba,pt %xcc, done2 ;\
ldd [ADDR + CPU_TMP1], %f62 ;\
done2:
.lddf_exception_not_aligned:
ldxa [MMU_SFAR]%asi, %g5 ! misaligned vaddr in %g5
#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
sethi %hi(fpu_exists), %g2 ! check fpu_exists
ld [%g2 + %lo(fpu_exists)], %g2
brz,a,pn %g2, 4f
nop
#endif
CPU_ADDR(%g1, %g4)
or %g0, 1, %g4
st %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
rdpr %tpc, %g2
lda [%g2]ASI_AIUP, %g6 ! get the user's lddf instruction
srl %g6, 23, %g1 ! using ldda or not?
and %g1, 1, %g1
brz,a,pt %g1, 2f ! check for ldda instruction
nop
srl %g6, 13, %g1 ! check immflag
and %g1, 1, %g1
rdpr %tstate, %g2 ! %tstate in %g2
brnz,a,pn %g1, 1f
srl %g2, 31, %g1 ! get asi from %tstate
srl %g6, 5, %g1 ! get asi from instruction
and %g1, 0xFF, %g1 ! imm_asi field
1:
cmp %g1, ASI_P ! primary address space
be,a,pt %icc, 2f
nop
cmp %g1, ASI_PNF ! primary no fault address space
be,a,pt %icc, 2f
nop
cmp %g1, ASI_S ! secondary address space
be,a,pt %icc, 2f
nop
cmp %g1, ASI_SNF ! secondary no fault address space
bne,a,pn %icc, 3f
nop
2:
lduwa [%g5]ASI_USER, %g7 ! get first half of misaligned data
add %g5, 4, %g5 ! increment misaligned data address
lduwa [%g5]ASI_USER, %g5 ! get second half of misaligned data
sllx %g7, 32, %g7
or %g5, %g7, %g5 ! combine data
CPU_ADDR(%g7, %g1) ! save data on a per-cpu basis
stx %g5, [%g7 + CPU_TMP1] ! save in cpu_tmp1
srl %g6, 25, %g3 ! %g6 has the instruction
and %g3, 0x1F, %g3 ! %g3 has rd
LDDF_REG(%g3, %g7, %g4)
CPU_ADDR(%g1, %g4)
st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
FAST_TRAP_DONE
3:
CPU_ADDR(%g1, %g4)
st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
4:
set T_USER, %g3 ! trap type in %g3
or %g3, T_LDDF_ALIGN, %g3
mov %g5, %g2 ! misaligned vaddr in %g2
set fpu_trap, %g1 ! goto C for the little and
ba,pt %xcc, sys_trap ! no fault little asi's
sub %g0, 1, %g4
.stdf_exception_not_aligned:
ldxa [MMU_SFAR]%asi, %g5 ! misaligned vaddr in %g5
#if defined(DEBUG) || defined(NEED_FPU_EXISTS)
sethi %hi(fpu_exists), %g7 ! check fpu_exists
ld [%g7 + %lo(fpu_exists)], %g3
brz,a,pn %g3, 4f
nop
#endif
CPU_ADDR(%g1, %g4)
or %g0, 1, %g4
st %g4, [%g1 + CPU_TL1_HDLR] ! set tl1_hdlr flag
rdpr %tpc, %g2
lda [%g2]ASI_AIUP, %g6 ! get the user's stdf instruction
srl %g6, 23, %g1 ! using stda or not?
and %g1, 1, %g1
brz,a,pt %g1, 2f ! check for stda instruction
nop
srl %g6, 13, %g1 ! check immflag
and %g1, 1, %g1
rdpr %tstate, %g2 ! %tstate in %g2
brnz,a,pn %g1, 1f
srl %g2, 31, %g1 ! get asi from %tstate
srl %g6, 5, %g1 ! get asi from instruction
and %g1, 0xFF, %g1 ! imm_asi field
1:
cmp %g1, ASI_P ! primary address space
be,a,pt %icc, 2f
nop
cmp %g1, ASI_S ! secondary address space
bne,a,pn %icc, 3f
nop
2:
srl %g6, 25, %g6
and %g6, 0x1F, %g6 ! %g6 has rd
CPU_ADDR(%g7, %g1)
STDF_REG(%g6, %g7, %g4) ! STDF_REG(REG, ADDR, TMP)
ldx [%g7 + CPU_TMP1], %g6
srlx %g6, 32, %g7
stuwa %g7, [%g5]ASI_USER ! first half
add %g5, 4, %g5 ! increment misaligned data address
stuwa %g6, [%g5]ASI_USER ! second half
CPU_ADDR(%g1, %g4)
st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
FAST_TRAP_DONE
3:
CPU_ADDR(%g1, %g4)
st %g0, [%g1 + CPU_TL1_HDLR] ! clear tl1_hdlr flag
4:
set T_USER, %g3 ! trap type in %g3
or %g3, T_STDF_ALIGN, %g3
mov %g5, %g2 ! misaligned vaddr in %g2
set fpu_trap, %g1 ! goto C for the little and
ba,pt %xcc, sys_trap ! nofault little asi's
sub %g0, 1, %g4
#ifdef DEBUG_USER_TRAPTRACECTL
.traptrace_freeze:
mov %l0, %g1 ; mov %l1, %g2 ; mov %l2, %g3 ; mov %l4, %g4
TT_TRACE_L(trace_win)
mov %g4, %l4 ; mov %g3, %l2 ; mov %g2, %l1 ; mov %g1, %l0
set trap_freeze, %g1
mov 1, %g2
st %g2, [%g1]
FAST_TRAP_DONE
.traptrace_unfreeze:
set trap_freeze, %g1
st %g0, [%g1]
mov %l0, %g1 ; mov %l1, %g2 ; mov %l2, %g3 ; mov %l4, %g4
TT_TRACE_L(trace_win)
mov %g4, %l4 ; mov %g3, %l2 ; mov %g2, %l1 ; mov %g1, %l0
FAST_TRAP_DONE
#endif
.getcc:
CPU_ADDR(%g1, %g2)
stx %o0, [%g1 + CPU_TMP1] ! save %o0
stx %o1, [%g1 + CPU_TMP2] ! save %o1
rdpr %tstate, %g3 ! get tstate
srlx %g3, PSR_TSTATE_CC_SHIFT, %o0 ! shift ccr to V8 psr
set PSR_ICC, %g2
and %o0, %g2, %o0 ! mask out the rest
srl %o0, PSR_ICC_SHIFT, %o0 ! right justify
rdpr %pstate, %o1
wrpr %o1, PSTATE_AG, %pstate ! get into normal globals
mov %o0, %g1 ! move ccr to normal %g1
wrpr %g0, %o1, %pstate ! back into alternate globals
ldx [%g1 + CPU_TMP1], %o0 ! restore %o0
ldx [%g1 + CPU_TMP2], %o1 ! restore %o1
FAST_TRAP_DONE
.setcc:
CPU_ADDR(%g1, %g2)
stx %o0, [%g1 + CPU_TMP1] ! save %o0
stx %o1, [%g1 + CPU_TMP2] ! save %o1
rdpr %pstate, %o0
wrpr %o0, PSTATE_AG, %pstate ! get into normal globals
mov %g1, %o1
wrpr %g0, %o0, %pstate ! back to alternates
sll %o1, PSR_ICC_SHIFT, %g2
set PSR_ICC, %g3
and %g2, %g3, %g2 ! mask out rest
sllx %g2, PSR_TSTATE_CC_SHIFT, %g2
rdpr %tstate, %g3 ! get tstate
srl %g3, 0, %g3 ! clear upper word
or %g3, %g2, %g3 ! or in new bits
wrpr %g3, %tstate
ldx [%g1 + CPU_TMP1], %o0 ! restore %o0
ldx [%g1 + CPU_TMP2], %o1 ! restore %o1
FAST_TRAP_DONE
.type .getpsr, #function
.getpsr:
rdpr %tstate, %g1 ! get tstate
srlx %g1, PSR_TSTATE_CC_SHIFT, %o0 ! shift ccr to V8 psr
set PSR_ICC, %g2
and %o0, %g2, %o0 ! mask out the rest
rd %fprs, %g1 ! get fprs
and %g1, FPRS_FEF, %g2 ! mask out dirty upper/lower
sllx %g2, PSR_FPRS_FEF_SHIFT, %g2 ! shift fef to V8 psr.ef
or %o0, %g2, %o0 ! or result into psr.ef
set V9_PSR_IMPLVER, %g2 ! SI assigned impl/ver: 0xef
or %o0, %g2, %o0 ! or psr.impl/ver
FAST_TRAP_DONE
SET_SIZE(.getpsr)
.type .setpsr, #function
.setpsr:
rdpr %tstate, %g1 ! get tstate
! setx TSTATE_V8_UBITS, %g2
or %g0, CCR_ICC, %g3
sllx %g3, TSTATE_CCR_SHIFT, %g2
andn %g1, %g2, %g1 ! zero current user bits
set PSR_ICC, %g2
and %g2, %o0, %g2 ! clear all but psr.icc bits
sllx %g2, PSR_TSTATE_CC_SHIFT, %g3 ! shift to tstate.ccr.icc
wrpr %g1, %g3, %tstate ! write tstate
set PSR_EF, %g2
and %g2, %o0, %g2 ! clear all but fp enable bit
srlx %g2, PSR_FPRS_FEF_SHIFT, %g4 ! shift ef to V9 fprs.fef
wr %g0, %g4, %fprs ! write fprs
CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1
ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
ldn [%g2 + T_LWP], %g3 ! load klwp pointer
ldn [%g3 + LWP_FPU], %g2 ! get lwp_fpu pointer
stuw %g4, [%g2 + FPU_FPRS] ! write fef value to fpu_fprs
srlx %g4, 2, %g4 ! shift fef value to bit 0
stub %g4, [%g2 + FPU_EN] ! write fef value to fpu_en
FAST_TRAP_DONE
SET_SIZE(.setpsr)
.type .getlgrp, #function
.getlgrp:
CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
ld [%g1 + CPU_ID], %o0 ! load cpu_id
ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
ldn [%g2 + T_LPL], %g2 ! load lpl pointer
ld [%g2 + LPL_LGRPID], %g1 ! load lpl_lgrpid
sra %g1, 0, %o1
FAST_TRAP_DONE
SET_SIZE(.getlgrp)
ENTRY_NP(syscall_trap_4x)
CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
ldn [%g2 + T_LWP], %g2 ! load klwp pointer
ld [%g2 + PCB_TRAP0], %g2 ! lwp->lwp_pcb.pcb_trap0addr
brz,pn %g2, 1f ! has it been set?
st %l0, [%g1 + CPU_TMP1] ! delay - save some locals
st %l1, [%g1 + CPU_TMP2]
rdpr %tnpc, %l1 ! save old tnpc
wrpr %g0, %g2, %tnpc ! setup tnpc
rdpr %pstate, %l0
wrpr %l0, PSTATE_AG, %pstate ! switch to normal globals
mov %l1, %g6 ! pass tnpc to user code in %g6
wrpr %l0, %g0, %pstate ! switch back to alternate globals
! Note that %g1 still contains CPU struct addr
ld [%g1 + CPU_TMP2], %l1 ! restore locals
ld [%g1 + CPU_TMP1], %l0
FAST_TRAP_DONE_CHK_INTR
1:
mov %g1, %l0
st %l1, [%g1 + CPU_TMP2]
rdpr %pstate, %l1
wrpr %l1, PSTATE_AG, %pstate
!
! check for old syscall mmap which is the only different one which
! must be the same. Others are handled in the compatibility library.
!
cmp %g1, OSYS_mmap ! compare to old 4.x mmap
movz %icc, SYS_mmap, %g1
wrpr %g0, %l1, %pstate
ld [%l0 + CPU_TMP2], %l1 ! restore locals
ld [%l0 + CPU_TMP1], %l0
SYSCALL(syscall_trap32)
SET_SIZE(syscall_trap_4x)
ENTRY_NP(set_trap0_addr)
CPU_ADDR(%g1, %g2) ! load CPU struct addr to %g1 using %g2
ldn [%g1 + CPU_THREAD], %g2 ! load thread pointer
ldn [%g2 + T_LWP], %g2 ! load klwp pointer
st %l0, [%g1 + CPU_TMP1] ! save some locals
st %l1, [%g1 + CPU_TMP2]
rdpr %pstate, %l0
wrpr %l0, PSTATE_AG, %pstate
mov %g1, %l1
wrpr %g0, %l0, %pstate
andn %l1, 3, %l1 ! force alignment
st %l1, [%g2 + PCB_TRAP0] ! lwp->lwp_pcb.pcb_trap0addr
ld [%g1 + CPU_TMP1], %l0 ! restore locals
ld [%g1 + CPU_TMP2], %l1
FAST_TRAP_DONE
SET_SIZE(set_trap0_addr)
.type mmu_trap_tl1, #function
mmu_trap_tl1:
#ifdef TRAPTRACE
TRACE_PTR(%g5, %g6)
GET_TRACE_TICK(%g6, %g7)
stxa %g6, [%g5 + TRAP_ENT_TICK]%asi
rdpr %tl, %g6
stha %g6, [%g5 + TRAP_ENT_TL]%asi
rdpr %tt, %g6
stha %g6, [%g5 + TRAP_ENT_TT]%asi
rdpr %tstate, %g6
stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi
stna %sp, [%g5 + TRAP_ENT_SP]%asi
stna %g0, [%g5 + TRAP_ENT_TR]%asi
rdpr %tpc, %g6
stna %g6, [%g5 + TRAP_ENT_TPC]%asi
set MMU_SFAR, %g6
ldxa [%g6]ASI_DMMU, %g6
stxa %g6, [%g5 + TRAP_ENT_F1]%asi
CPU_PADDR(%g7, %g6);
add %g7, CPU_TL1_HDLR, %g7
lda [%g7]ASI_MEM, %g6
stxa %g6, [%g5 + TRAP_ENT_F2]%asi
set 0xdeadbeef, %g6
stna %g6, [%g5 + TRAP_ENT_F3]%asi
stna %g6, [%g5 + TRAP_ENT_F4]%asi
TRACE_NEXT(%g5, %g6, %g7)
#endif
GET_CPU_IMPL(%g5)
cmp %g5, PANTHER_IMPL
bne mmu_trap_tl1_4
nop
rdpr %tt, %g5
cmp %g5, T_DATA_EXCEPTION
bne mmu_trap_tl1_4
nop
wr %g0, ASI_DMMU, %asi
ldxa [MMU_SFSR]%asi, %g5
mov 1, %g6
sllx %g6, PN_SFSR_PARITY_SHIFT, %g6
andcc %g5, %g6, %g0
bz mmu_trap_tl1_4
ldxa [MMU_TAG_ACCESS]%asi, %g2
mov %g5, %g3
ba,pt %xcc, .mmu_exception_is_tlb_parity
mov T_DATA_EXCEPTION, %g1
mmu_trap_tl1_4:
CPU_PADDR(%g7, %g6);
add %g7, CPU_TL1_HDLR, %g7 ! %g7 = &cpu_m.tl1_hdlr (PA)
lda [%g7]ASI_MEM, %g6
brz,a,pt %g6, 1f
nop
CPU_ADDR(%g6, %g5)
add %g6, CPU_TL1_HDLR, %g6 ! %g6 = &cpu_m.tl1_hdlr (VA)
GET_CPU_IMPL(%g5)
cmp %g5, CHEETAH_IMPL
bl,pt %icc, 3f
cmp %g5, SPITFIRE_IMPL
stxa %g0, [%g7]ASI_DC_INVAL
membar #Sync
ba,pt %xcc, 2f
nop
3:
bl,pt %icc, 2f
sethi %hi(dcache_line_mask), %g5
ld [%g5 + %lo(dcache_line_mask)], %g5
and %g6, %g5, %g5
stxa %g0, [%g5]ASI_DC_TAG
membar #Sync
2:
sta %g0, [%g7]ASI_MEM
SWITCH_GLOBALS ! back to mmu globals
ba,a,pt %xcc, sfmmu_mmu_trap ! handle page faults
1:
rdpr %tt, %g5
rdpr %tl, %g7
sub %g7, 1, %g6
wrpr %g6, %tl
rdpr %tt, %g6
wrpr %g7, %tl
and %g6, WTRAP_TTMASK, %g6
cmp %g6, WTRAP_TYPE
bne,a,pn %xcc, ptl1_panic
mov PTL1_BAD_MMUTRAP, %g1
rdpr %tpc, %g7
set trap_table, %g6
cmp %g7, %g6
blt,a,pn %xcc, ptl1_panic
mov PTL1_BAD_MMUTRAP, %g1
set etrap_table, %g6
cmp %g7, %g6
bge,a,pn %xcc, ptl1_panic
mov PTL1_BAD_MMUTRAP, %g1
cmp %g5, T_ALIGNMENT
move %icc, MMU_SFAR, %g6
movne %icc, MMU_TAG_ACCESS, %g6
ldxa [%g6]ASI_DMMU, %g6
andn %g7, WTRAP_ALIGN, %g7
add %g7, WTRAP_FAULTOFF, %g7
wrpr %g0, %g7, %tnpc
done
SET_SIZE(mmu_trap_tl1)
.global kmdb_trap
.align 8
kmdb_trap:
ba,a trap_table0
jmp %g1 + 0
nop
.global kmdb_trap_tl1
.align 8
kmdb_trap_tl1:
ba,a trap_table0
jmp %g1 + 0
nop
.global obp_bpt
.align 8
obp_bpt:
NOT
.global kctx_obp_bpt
kctx_obp_bpt:
set obp_bpt, %g2
1:
#ifndef _OPL
mov MMU_PCONTEXT, %g1
ldxa [%g1]ASI_DMMU, %g1
srlx %g1, CTXREG_NEXT_SHIFT, %g3
brz,pt %g3, 3f ! nucleus pgsz is 0, no problem
sllx %g3, CTXREG_NEXT_SHIFT, %g3
set CTXREG_CTX_MASK, %g4 ! check Pcontext
btst %g4, %g1
bz,a,pt %xcc, 2f
clr %g3 ! kernel: PCONTEXT=0
xor %g3, %g1, %g3 ! user: clr N_pgsz0/1 bits
2:
set DEMAP_ALL_TYPE, %g1
stxa %g0, [%g1]ASI_DTLB_DEMAP
stxa %g0, [%g1]ASI_ITLB_DEMAP
mov MMU_PCONTEXT, %g1
stxa %g3, [%g1]ASI_DMMU
membar #Sync
sethi %hi(FLUSH_ADDR), %g1
flush %g1 ! flush required by immu
#endif
3:
jmp %g2
nop
#ifdef TRAPTRACE
trace_gen:
TRACE_PTR(%g3, %g6)
GET_TRACE_TICK(%g6, %g4)
stxa %g6, [%g3 + TRAP_ENT_TICK]%asi
rdpr %tl, %g6
stha %g6, [%g3 + TRAP_ENT_TL]%asi
rdpr %tt, %g6
stha %g6, [%g3 + TRAP_ENT_TT]%asi
rdpr %tstate, %g6
stxa %g6, [%g3 + TRAP_ENT_TSTATE]%asi
stna %sp, [%g3 + TRAP_ENT_SP]%asi
rdpr %tpc, %g6
stna %g6, [%g3 + TRAP_ENT_TPC]%asi
TRACE_NEXT(%g3, %g4, %g5)
jmp %g7 + 4
nop
trace_win:
TRACE_WIN_INFO(0, %l0, %l1, %l2)
! Keep the locals as clean as possible, caller cleans %l4
clr %l2
clr %l1
jmp %l4 + 4
clr %l0
! Do not disturb %g5, it will be used after the trace
ALTENTRY(trace_tsbhit)
TRACE_TSBHIT(0)
jmp %g7 + 4
nop
.global trace_tsbmiss
trace_tsbmiss:
membar #Sync
sethi %hi(FLUSH_ADDR), %g6
flush %g6
TRACE_PTR(%g5, %g6)
stxa %g2, [%g5 + TRAP_ENT_SP]%asi ! tag access
stxa %g4, [%g5 + TRAP_ENT_F1]%asi ! tsb tag
GET_TRACE_TICK(%g6, %g4)
stxa %g6, [%g5 + TRAP_ENT_TICK]%asi
rdpr %tnpc, %g6
stxa %g6, [%g5 + TRAP_ENT_F2]%asi
stna %g1, [%g5 + TRAP_ENT_F3]%asi ! tsb8k pointer
srlx %g1, 32, %g6
stna %g6, [%g5 + TRAP_ENT_F4]%asi ! huh?
rdpr %tpc, %g6
stna %g6, [%g5 + TRAP_ENT_TPC]%asi
rdpr %tl, %g6
stha %g6, [%g5 + TRAP_ENT_TL]%asi
rdpr %tt, %g6
or %g6, TT_MMU_MISS, %g4
stha %g4, [%g5 + TRAP_ENT_TT]%asi
cmp %g6, FAST_IMMU_MISS_TT
be,a %icc, 1f
ldxa [%g0]ASI_IMMU, %g6
ldxa [%g0]ASI_DMMU, %g6
1: stxa %g6, [%g5 + TRAP_ENT_TSTATE]%asi ! tag target
stxa %g3, [%g5 + TRAP_ENT_TR]%asi ! tsb4m pointer
TRACE_NEXT(%g5, %g4, %g6)
jmp %g7 + 4
nop
trace_dataprot:
membar #Sync
sethi %hi(FLUSH_ADDR), %g6
flush %g6
TRACE_PTR(%g1, %g6)
GET_TRACE_TICK(%g6, %g5)
stxa %g6, [%g1 + TRAP_ENT_TICK]%asi
rdpr %tpc, %g6
stna %g6, [%g1 + TRAP_ENT_TPC]%asi
rdpr %tstate, %g6
stxa %g6, [%g1 + TRAP_ENT_TSTATE]%asi
stxa %g2, [%g1 + TRAP_ENT_SP]%asi ! tag access reg
stxa %g0, [%g1 + TRAP_ENT_TR]%asi
stxa %g0, [%g1 + TRAP_ENT_F1]%asi
stxa %g0, [%g1 + TRAP_ENT_F2]%asi
stxa %g0, [%g1 + TRAP_ENT_F3]%asi
stxa %g0, [%g1 + TRAP_ENT_F4]%asi
rdpr %tl, %g6
stha %g6, [%g1 + TRAP_ENT_TL]%asi
rdpr %tt, %g6
stha %g6, [%g1 + TRAP_ENT_TT]%asi
TRACE_NEXT(%g1, %g4, %g5)
jmp %g7 + 4
nop
#endif
.align 32
.global pil15_epilogue
pil15_epilogue:
ba pil_interrupt_common
nop
.align 32
.global fast_trap_done, fast_trap_done_chk_intr
fast_trap_done:
rdpr %tpc, %g5
sethi %hi(0xffffc000), %g6 ! 1's complement of 0x3fff
andncc %g5, %g6, %g0 ! check lower 14 bits of %tpc
bz,a,pn %icc, 1f ! branch if zero (lower 32 bits only)
ldxa [%g0]ASI_INTR_RECEIVE_STATUS, %g5
done
ALTENTRY(fast_trap_done_check_interrupts)
fast_trap_done_chk_intr:
ldxa [%g0]ASI_INTR_RECEIVE_STATUS, %g5
1: rd SOFTINT, %g6
and %g5, IRSR_BUSY, %g5
orcc %g5, %g6, %g0
bnz,pn %xcc, 2f ! branch if any pending intr
nop
done
2:
rdpr %tnpc, %g5
wrpr %g0, %g5, %tpc
add %g5, 4, %g5
wrpr %g0, %g5, %tnpc
set fast_trap_dummy_call, %g1
ba,pt %xcc, sys_trap
mov -1, %g4
fast_trap_dummy_call:
retl
nop
#define BRAND_CALLBACK(callback_id) \
CPU_ADDR(%g2, %g1) ;\
ldn [%g2 + CPU_THREAD], %g3 ;\
ldn [%g3 + T_PROCP], %g3 ;\
ldn [%g3 + P_BRAND], %g3 ;\
brz %g3, 1f ;\
nop ;\
ldn [%g3 + B_MACHOPS], %g3 ;\
ldn [%g3 + (callback_id << 3)], %g3 ;\
brz %g3, 1f ;\
\
mov %pc, %g1 ;\
add %g1, 16, %g1 ;\
jmp %g3 ;\
nop ;\
1:
ENTRY_NP(syscall_wrapper32)
BRAND_CALLBACK(BRAND_CB_SYSCALL32)
SYSCALL_NOTT(syscall_trap32)
SET_SIZE(syscall_wrapper32)
ENTRY_NP(syscall_wrapper)
BRAND_CALLBACK(BRAND_CB_SYSCALL)
SYSCALL_NOTT(syscall_trap)
SET_SIZE(syscall_wrapper)
