#include <sys/asm_linkage.h>
#include <sys/asm_misc.h>
#include <sys/trap.h>
#include <sys/psw.h>
#include <sys/regset.h>
#include <sys/privregs.h>
#include <sys/dtrace.h>
#include <sys/x86_archext.h>
#include <sys/traptrace.h>
#include <sys/machparam.h>
#include "assym.h"
#if defined(__xpv)
#define NPTRAP_NOERR(trapno) \
pushq $0; \
pushq $trapno
#define TRAP_NOERR(trapno) \
XPV_TRAP_POP; \
NPTRAP_NOERR(trapno)
#define TRAP_ERR(trapno) \
XPV_TRAP_POP; \
pushq $trapno
#else
#define TRAP_NOERR(trapno) \
push $0; \
push $trapno
#define NPTRAP_NOERR(trapno) TRAP_NOERR(trapno)
#define TRAP_ERR(trapno) \
push $trapno
#endif
.data
DGDEF3(dblfault_stack0, DEFAULTSTKSZ, MMU_PAGESIZE)
.fill DEFAULTSTKSZ, 1, 0
DGDEF3(nmi_stack0, DEFAULTSTKSZ, MMU_PAGESIZE)
.fill DEFAULTSTKSZ, 1, 0
DGDEF3(mce_stack0, DEFAULTSTKSZ, MMU_PAGESIZE)
.fill DEFAULTSTKSZ, 1, 0
ENTRY_NP(div0trap)
TRAP_NOERR(T_ZERODIV)
jmp cmntrap
SET_SIZE(div0trap)
ENTRY_NP(dbgtrap)
TRAP_NOERR(T_SGLSTP)
#if !defined(__xpv)
pushq %r11
leaq sys_sysenter(%rip), %r11
cmpq %r11, 24(%rsp)
je 1f
leaq brand_sys_sysenter(%rip), %r11
cmpq %r11, 24(%rsp)
je 1f
leaq tr_sys_sysenter(%rip), %r11
cmpq %r11, 24(%rsp)
je 1f
leaq tr_brand_sys_sysenter(%rip), %r11
cmpq %r11, 24(%rsp)
jne 2f
1: swapgs
2: lfence
popq %r11
#endif
INTR_PUSH
#if defined(__xpv)
movl $6, %edi
call kdi_dreg_get
movq %rax, %r15
movl $6, %edi
movl $0, %esi
call kdi_dreg_set
#else
movq %db6, %r15
xorl %eax, %eax
movq %rax, %db6
#endif
jmp cmntrap_pushed
SET_SIZE(dbgtrap)
#if !defined(__xpv)
#define SET_CPU_GSBASE \
subq $REGOFF_TRAPNO, %rsp; \
movq %rax, REGOFF_RAX(%rsp); \
movq %rbx, REGOFF_RBX(%rsp); \
movq %rcx, REGOFF_RCX(%rsp); \
movq %rdx, REGOFF_RDX(%rsp); \
movq %rbp, REGOFF_RBP(%rsp); \
movq %rsp, %rbp; \
subq $16, %rsp; \
sgdt 6(%rsp); \
movq 8(%rsp), %rcx; \
xorl %ebx, %ebx; \
leaq cpu(%rip), %rdx; \
1: \
movq (%rdx, %rbx, CLONGSIZE), %rax; \
cmpq $0x0, %rax; \
je 2f; \
cmpq %rcx, CPU_GDT(%rax); \
je 3f; \
2: \
incl %ebx; \
cmpl $NCPU, %ebx; \
jb 1b; \
\
3: \
movl $MSR_AMD_KGSBASE, %ecx; \
cmpw $KCS_SEL, REGOFF_CS(%rbp); \
jne 4f; \
movl $MSR_AMD_GSBASE, %ecx; \
mfence; \
4: \
movq %rax, %rdx; \
shrq $32, %rdx; \
wrmsr; \
movq REGOFF_RDX(%rbp), %rdx; \
movq REGOFF_RCX(%rbp), %rcx; \
movq REGOFF_RBX(%rbp), %rbx; \
movq REGOFF_RAX(%rbp), %rax; \
movq %rbp, %rsp; \
movq REGOFF_RBP(%rsp), %rbp; \
addq $REGOFF_TRAPNO, %rsp
#else
#define SET_CPU_GSBASE
#endif
ENTRY_NP(nmiint)
TRAP_NOERR(T_NMIFLT)
SET_CPU_GSBASE
INTR_PUSH
INTGATE_INIT_KERNEL_FLAGS
TRACE_PTR(%r12, %rax, %eax, %rdx, $TT_TRAP)
TRACE_REGS(%r12, %rsp, %rax, %rbx)
TRACE_STAMP(%r12)
movq %rsp, %rbp
movq %rbp, %rdi
call av_dispatch_nmivect
INTR_POP
call x86_md_clear
jmp tr_iret_auto
SET_SIZE(nmiint)
ENTRY_NP(brktrap)
XPV_TRAP_POP
cmpw $KCS_SEL, 8(%rsp)
jne bp_user
decq (%rsp)
push $1
jmp ud_kernel
bp_user:
NPTRAP_NOERR(T_BPTFLT)
jmp dtrace_trap
SET_SIZE(brktrap)
ENTRY_NP(ovflotrap)
TRAP_NOERR(T_OVFLW)
jmp cmntrap
SET_SIZE(ovflotrap)
ENTRY_NP(boundstrap)
TRAP_NOERR(T_BOUNDFLT)
jmp cmntrap
SET_SIZE(boundstrap)
ENTRY_NP(invoptrap)
XPV_TRAP_POP
cmpw $KCS_SEL, 8(%rsp)
jne ud_user
#if defined(__xpv)
movb $0, 12(%rsp)
#endif
push $0
ud_kernel:
push $0xdddd
INTR_PUSH
movq REGOFF_RIP(%rsp), %rdi
movq REGOFF_RSP(%rsp), %rsi
movq REGOFF_RAX(%rsp), %rdx
pushq (%rsi)
movq %rsp, %rsi
subq $8, %rsp
call dtrace_invop
ALTENTRY(dtrace_invop_callsite)
addq $16, %rsp
cmpl $DTRACE_INVOP_PUSHL_EBP, %eax
je ud_push
cmpl $DTRACE_INVOP_LEAVE, %eax
je ud_leave
cmpl $DTRACE_INVOP_NOP, %eax
je ud_nop
cmpl $DTRACE_INVOP_RET, %eax
je ud_ret
jmp ud_trap
ud_push:
INTR_POP
subq $16, %rsp
pushq %rax
movq 24(%rsp), %rax
addq $1, %rax
movq %rax, 8(%rsp)
movq 32(%rsp), %rax
movq %rax, 16(%rsp)
movq 40(%rsp), %rax
movq %rax, 24(%rsp)
movq 48(%rsp), %rax
subq $8, %rax
movq %rax, 32(%rsp)
movq 56(%rsp), %rax
movq %rax, 40(%rsp)
movq 32(%rsp), %rax
movq %rbp, (%rax)
popq %rax
jmp tr_iret_kernel
ud_leave:
INTR_POP
pushq %rax
movq 8(%rsp), %rax
addq $1, %rax
movq %rax, 8(%rsp)
movq (%rbp), %rax
addq $8, %rbp
movq %rbp, 32(%rsp)
movq %rax, %rbp
popq %rax
jmp tr_iret_kernel
ud_nop:
INTR_POP
incq (%rsp)
jmp tr_iret_kernel
ud_ret:
INTR_POP
pushq %rax
movq 32(%rsp), %rax
movq (%rax), %rax
movq %rax, 8(%rsp)
addq $8, 32(%rsp)
popq %rax
jmp tr_iret_kernel
ud_trap:
cmpq $0, REGOFF_ERR(%rsp)
je ud_ud
incq REGOFF_RIP(%rsp)
addq $REGOFF_RIP, %rsp
NPTRAP_NOERR(T_BPTFLT)
jmp cmntrap
ud_ud:
addq $REGOFF_RIP, %rsp
ud_user:
NPTRAP_NOERR(T_ILLINST)
jmp cmntrap
SET_SIZE(invoptrap)
ENTRY_NP(ndptrap)
TRAP_NOERR(T_NOEXTFLT)
SET_CPU_GSBASE
jmp cmntrap
SET_SIZE(ndptrap)
#if !defined(__xpv)
ENTRY_NP(syserrtrap)
pushq $T_DBLFLT
SET_CPU_GSBASE
pushq %rax
subq $DESCTBR_SIZE, %rsp
sidt (%rsp)
movq %gs:CPU_IDT, %rax
cmpq %rax, DTR_BASE(%rsp)
je 1f
movq %rax, DTR_BASE(%rsp)
movw $_MUL(NIDT, GATE_DESC_SIZE), DTR_LIMIT(%rsp)
lidt (%rsp)
movl $1, nopanicdebug
1: addq $DESCTBR_SIZE, %rsp
popq %rax
DFTRAP_PUSH
#ifdef TRAPTRACE
leaq trap_trace_freeze(%rip), %r11
incl (%r11)
#endif
ENABLE_INTR_FLAGS
movq %rsp, %rdi
xorl %esi, %esi
xorl %edx, %edx
call trap
SET_SIZE(syserrtrap)
#endif
ENTRY_NP(invtsstrap)
TRAP_ERR(T_TSSFLT)
jmp cmntrap
SET_SIZE(invtsstrap)
ENTRY_NP(segnptrap)
TRAP_ERR(T_SEGFLT)
SET_CPU_GSBASE
jmp cmntrap
SET_SIZE(segnptrap)
ENTRY_NP(stktrap)
TRAP_ERR(T_STKFLT)
SET_CPU_GSBASE
jmp cmntrap
SET_SIZE(stktrap)
ENTRY_NP(gptrap)
TRAP_ERR(T_GPFLT)
SET_CPU_GSBASE
jmp cmntrap
SET_SIZE(gptrap)
ENTRY_NP(pftrap)
TRAP_ERR(T_PGFLT)
INTR_PUSH
#if defined(__xpv)
movq %gs:CPU_VCPU_INFO, %r15
movq VCPU_INFO_ARCH_CR2(%r15), %r15
#else
movq %cr2, %r15
#endif
jmp cmntrap_pushed
SET_SIZE(pftrap)
ENTRY_NP(resvtrap)
TRAP_NOERR(T_RESVTRAP)
jmp cmntrap
SET_SIZE(resvtrap)
ENTRY_NP(ndperr)
TRAP_NOERR(T_EXTERRFLT)
jmp cmninttrap
SET_SIZE(ndperr)
ENTRY_NP(achktrap)
TRAP_ERR(T_ALIGNMENT)
jmp cmntrap
SET_SIZE(achktrap)
.globl cmi_mca_trap
ENTRY_NP(mcetrap)
TRAP_NOERR(T_MCE)
SET_CPU_GSBASE
INTR_PUSH
INTGATE_INIT_KERNEL_FLAGS
TRACE_PTR(%rdi, %rbx, %ebx, %rcx, $TT_TRAP)
TRACE_REGS(%rdi, %rsp, %rbx, %rcx)
TRACE_STAMP(%rdi)
movq %rsp, %rbp
movq %rsp, %rdi
call cmi_mca_trap
jmp _sys_rtt
SET_SIZE(mcetrap)
ENTRY_NP(xmtrap)
TRAP_NOERR(T_SIMDFPE)
jmp cmninttrap
SET_SIZE(xmtrap)
ENTRY_NP(invaltrap)
TRAP_NOERR(T_INVALTRAP)
jmp cmntrap
SET_SIZE(invaltrap)
.globl fasttable
ENTRY_NP(fasttrap)
cmpl $T_LASTFAST, %eax
ja 1f
orl %eax, %eax
leaq fasttable(%rip), %r11
leaq (%r11, %rax, CLONGSIZE), %r11
movq (%r11), %r11
INDIRECT_JMP_REG(r11)
1:
XPV_TRAP_POP
subq $2, (%rsp)
pushq $_CONST(_MUL(T_FASTTRAP, GATE_DESC_SIZE) + 2)
#if defined(__xpv)
pushq %r11
pushq %rcx
#endif
jmp gptrap
SET_SIZE(fasttrap)
ENTRY_NP(dtrace_ret)
TRAP_NOERR(T_DTRACE_RET)
jmp dtrace_trap
SET_SIZE(dtrace_ret)
ENTRY_NP(fast_null)
XPV_TRAP_POP
orq $PS_C, 24(%rsp)
call x86_md_clear
jmp tr_iret_auto
SET_SIZE(fast_null)
#define MKIVCT(n) \
ENTRY_NP(ivct##n) \
push $0; \
push $n - 0x20; \
jmp cmnint; \
SET_SIZE(ivct##n)
MKIVCT(32)
MKIVCT(33)
MKIVCT(34)
MKIVCT(35)
MKIVCT(36)
MKIVCT(37)
MKIVCT(38)
MKIVCT(39)
MKIVCT(40)
MKIVCT(41)
MKIVCT(42)
MKIVCT(43)
MKIVCT(44)
MKIVCT(45)
MKIVCT(46)
MKIVCT(47)
MKIVCT(48)
MKIVCT(49)
MKIVCT(50)
MKIVCT(51)
MKIVCT(52)
MKIVCT(53)
MKIVCT(54)
MKIVCT(55)
MKIVCT(56)
MKIVCT(57)
MKIVCT(58)
MKIVCT(59)
MKIVCT(60)
MKIVCT(61)
MKIVCT(62)
MKIVCT(63)
MKIVCT(64)
MKIVCT(65)
MKIVCT(66)
MKIVCT(67)
MKIVCT(68)
MKIVCT(69)
MKIVCT(70)
MKIVCT(71)
MKIVCT(72)
MKIVCT(73)
MKIVCT(74)
MKIVCT(75)
MKIVCT(76)
MKIVCT(77)
MKIVCT(78)
MKIVCT(79)
MKIVCT(80)
MKIVCT(81)
MKIVCT(82)
MKIVCT(83)
MKIVCT(84)
MKIVCT(85)
MKIVCT(86)
MKIVCT(87)
MKIVCT(88)
MKIVCT(89)
MKIVCT(90)
MKIVCT(91)
MKIVCT(92)
MKIVCT(93)
MKIVCT(94)
MKIVCT(95)
MKIVCT(96)
MKIVCT(97)
MKIVCT(98)
MKIVCT(99)
MKIVCT(100)
MKIVCT(101)
MKIVCT(102)
MKIVCT(103)
MKIVCT(104)
MKIVCT(105)
MKIVCT(106)
MKIVCT(107)
MKIVCT(108)
MKIVCT(109)
MKIVCT(110)
MKIVCT(111)
MKIVCT(112)
MKIVCT(113)
MKIVCT(114)
MKIVCT(115)
MKIVCT(116)
MKIVCT(117)
MKIVCT(118)
MKIVCT(119)
MKIVCT(120)
MKIVCT(121)
MKIVCT(122)
MKIVCT(123)
MKIVCT(124)
MKIVCT(125)
MKIVCT(126)
MKIVCT(127)
MKIVCT(128)
MKIVCT(129)
MKIVCT(130)
MKIVCT(131)
MKIVCT(132)
MKIVCT(133)
MKIVCT(134)
MKIVCT(135)
MKIVCT(136)
MKIVCT(137)
MKIVCT(138)
MKIVCT(139)
MKIVCT(140)
MKIVCT(141)
MKIVCT(142)
MKIVCT(143)
MKIVCT(144)
MKIVCT(145)
MKIVCT(146)
MKIVCT(147)
MKIVCT(148)
MKIVCT(149)
MKIVCT(150)
MKIVCT(151)
MKIVCT(152)
MKIVCT(153)
MKIVCT(154)
MKIVCT(155)
MKIVCT(156)
MKIVCT(157)
MKIVCT(158)
MKIVCT(159)
MKIVCT(160)
MKIVCT(161)
MKIVCT(162)
MKIVCT(163)
MKIVCT(164)
MKIVCT(165)
MKIVCT(166)
MKIVCT(167)
MKIVCT(168)
MKIVCT(169)
MKIVCT(170)
MKIVCT(171)
MKIVCT(172)
MKIVCT(173)
MKIVCT(174)
MKIVCT(175)
MKIVCT(176)
MKIVCT(177)
MKIVCT(178)
MKIVCT(179)
MKIVCT(180)
MKIVCT(181)
MKIVCT(182)
MKIVCT(183)
MKIVCT(184)
MKIVCT(185)
MKIVCT(186)
MKIVCT(187)
MKIVCT(188)
MKIVCT(189)
MKIVCT(190)
MKIVCT(191)
MKIVCT(192)
MKIVCT(193)
MKIVCT(194)
MKIVCT(195)
MKIVCT(196)
MKIVCT(197)
MKIVCT(198)
MKIVCT(199)
MKIVCT(200)
MKIVCT(201)
MKIVCT(202)
MKIVCT(203)
MKIVCT(204)
MKIVCT(205)
MKIVCT(206)
MKIVCT(207)
MKIVCT(208)
MKIVCT(209)
MKIVCT(210)
MKIVCT(211)
MKIVCT(212)
MKIVCT(213)
MKIVCT(214)
MKIVCT(215)
MKIVCT(216)
MKIVCT(217)
MKIVCT(218)
MKIVCT(219)
MKIVCT(220)
MKIVCT(221)
MKIVCT(222)
MKIVCT(223)
MKIVCT(224)
MKIVCT(225)
MKIVCT(226)
MKIVCT(227)
MKIVCT(228)
MKIVCT(229)
MKIVCT(230)
MKIVCT(231)
MKIVCT(232)
MKIVCT(233)
MKIVCT(234)
MKIVCT(235)
MKIVCT(236)
MKIVCT(237)
MKIVCT(238)
MKIVCT(239)
MKIVCT(240)
MKIVCT(241)
MKIVCT(242)
MKIVCT(243)
MKIVCT(244)
MKIVCT(245)
MKIVCT(246)
MKIVCT(247)
MKIVCT(248)
MKIVCT(249)
MKIVCT(250)
MKIVCT(251)
MKIVCT(252)
MKIVCT(253)
MKIVCT(254)
MKIVCT(255)