/*
 * Copyright 2018, Jérôme Duval, jerome.duval@gmail.com.
 * Copyright 2012, Alex Smith, alex@alex-smith.me.uk.
 * Distributed under the terms of the MIT License.
 */


#include <asm_defs.h>

#include <thread_types.h>

#include <arch/x86/descriptors.h>
#include <arch/x86/arch_altcodepatch.h>
#include <arch/x86/arch_cpu.h>
#include <arch/x86/arch_kernel.h>

#include "asm_offsets.h"
#include "syscall_numbers.h"
#include "syscall_table.h"


// Push the remainder of the interrupt frame onto the stack.
#define PUSH_IFRAME_BOTTOM(iframeType)  \
        push    %rax;   /* orig_rax */          \
        push    %rax;                                           \
        push    %rbx;                                           \
        push    %rcx;                                           \
        push    %rdx;                                           \
        push    %rdi;                                           \
        push    %rsi;                                           \
        push    %rbp;                                           \
        push    %r8;                                            \
        push    %r9;                                            \
        push    %r10;                                           \
        push    %r11;                                           \
        push    %r12;                                           \
        push    %r13;                                           \
        push    %r14;                                           \
        push    %r15;                                           \
        pushq   $0;                                                     \
        push    $iframeType;


// Restore the interrupt frame.
#define RESTORE_IFRAME()                                \
        add             $16, %rsp;                                      \
        pop             %r15;                                           \
        pop             %r14;                                           \
        pop             %r13;                                           \
        pop             %r12;                                           \
        pop             %r11;                                           \
        pop             %r10;                                           \
        pop             %r9;                                            \
        pop             %r8;                                            \
        pop             %rbp;                                           \
        pop             %rsi;                                           \
        pop             %rdi;                                           \
        pop             %rdx;                                           \
        pop             %rcx;                                           \
        pop             %rbx;                                           \
        pop             %rax;                                           \
        addq    $24, %rsp;


// The macros below require R12 to contain the current thread pointer. R12 is
// callee-save so will be preserved through all function calls and only needs
// to be obtained once.

#define LOCK_THREAD_TIME()                                                                              \
        leaq    THREAD_time_lock(%r12), %rdi;                                           \
        call    acquire_spinlock;

#define UNLOCK_THREAD_TIME()                                                                    \
        leaq    THREAD_time_lock(%r12), %rdi;                                           \
        call    release_spinlock;                                                                       \

#define UPDATE_THREAD_USER_TIME()                                                               \
        LOCK_THREAD_TIME()                                                                                      \
                                                                                                                                \
        call    system_time;                                                                            \
        movq    %rax, %r13;                                                                                     \
                                                                                                                                \
        /* thread->user_time += now - thread->last_time; */                     \
        subq    THREAD_last_time(%r12), %rax;                                           \
        addq    %rax, THREAD_user_time(%r12);                                           \
                                                                                                                                \
        /* thread->last_time = now; */                                                          \
        movq    %r13, THREAD_last_time(%r12);                                           \
                                                                                                                                \
        /* thread->in_kernel = true; */                                                         \
        movb    $1, THREAD_in_kernel(%r12);                                                     \
                                                                                                                                \
        UNLOCK_THREAD_TIME()

#define UPDATE_THREAD_KERNEL_TIME()                                                             \
        LOCK_THREAD_TIME()                                                                                      \
                                                                                                                                \
        call    system_time;                                                                            \
        movq    %rax, %r13;                                                                                     \
                                                                                                                                \
        /* thread->kernel_time += now - thread->last_time; */           \
        subq    THREAD_last_time(%r12), %rax;                                           \
        addq    %rax, THREAD_kernel_time(%r12);                                         \
                                                                                                                                \
        /* thread->last_time = now; */                                                          \
        movq    %r13, THREAD_last_time(%r12);                                           \
                                                                                                                                \
        /* thread->in_kernel = false; */                                                        \
        movb    $0, THREAD_in_kernel(%r12);                                                     \
                                                                                                                                \
        UNLOCK_THREAD_TIME()

#define STOP_USER_DEBUGGING()                                                                   \
        testl   $(THREAD_FLAGS_BREAKPOINTS_INSTALLED                            \
                        | THREAD_FLAGS_SINGLE_STEP), THREAD_flags(%r12);        \
        jz              1f;                                                                                                     \
        call    x86_exit_user_debug_at_kernel_entry;                            \
  1:

#define CLEAR_FPU_STATE() \
        pxor %xmm0, %xmm0; \
        pxor %xmm1, %xmm1; \
        pxor %xmm2, %xmm2; \
        pxor %xmm3, %xmm3; \
        pxor %xmm4, %xmm4; \
        pxor %xmm5, %xmm5; \
        pxor %xmm6, %xmm6; \
        pxor %xmm7, %xmm7; \
        pxor %xmm8, %xmm8; \
        pxor %xmm9, %xmm9; \
        pxor %xmm10, %xmm10; \
        pxor %xmm11, %xmm11; \
        pxor %xmm12, %xmm12; \
        pxor %xmm13, %xmm13; \
        pxor %xmm14, %xmm14; \
        pxor %xmm15, %xmm15

// The following code defines the interrupt service routines for all 256
// interrupts. It creates a block of handlers, each 16 bytes, that the IDT
// initialization code just loops through.

// Interrupt with no error code, pushes a 0 error code.
#define DEFINE_ISR(nr)                                  \
        .align 16;                                                      \
        ASM_CLAC                                                        \
        push    $0;                                                     \
        push    $nr;                                            \
        jmp             intr_bottom;

// Interrupt with an error code.
#define DEFINE_ISR_E(nr)                                \
        .align 16;                                                      \
        ASM_CLAC                                                        \
        push    $nr;                                            \
        jmp             intr_bottom;

// Array of interrupt service routines.
.align 16
SYMBOL(isr_array):
        // Exceptions (0-19) and reserved interrupts (20-31).
        DEFINE_ISR(0)
        DEFINE_ISR(1)
        DEFINE_ISR(2)
        DEFINE_ISR(3)
        DEFINE_ISR(4)
        DEFINE_ISR(5)
        DEFINE_ISR(6)
        DEFINE_ISR(7)
        DEFINE_ISR_E(8)
        DEFINE_ISR(9)
        DEFINE_ISR_E(10)
        DEFINE_ISR_E(11)
        DEFINE_ISR_E(12)
        DEFINE_ISR_E(13)
        DEFINE_ISR_E(14)
        DEFINE_ISR(15)
        DEFINE_ISR(16)
        DEFINE_ISR_E(17)
        DEFINE_ISR(18)
        DEFINE_ISR(19)
        DEFINE_ISR(20)
        DEFINE_ISR(21)
        DEFINE_ISR(22)
        DEFINE_ISR(23)
        DEFINE_ISR(24)
        DEFINE_ISR(25)
        DEFINE_ISR(26)
        DEFINE_ISR(27)
        DEFINE_ISR(28)
        DEFINE_ISR(29)
        DEFINE_ISR(30)
        DEFINE_ISR(31)

        // User-defined ISRs (32-255) - none take an error code.
        .Lintr = 32
        .rept 224
                DEFINE_ISR(.Lintr)
                .Lintr = .Lintr+1
        .endr


// Common interrupt handling code.
STATIC_FUNCTION(intr_bottom):
        // Coming from user-mode requires special handling.
        testl   $3, 24(%rsp)
        jnz             intr_bottom_user

        // Push the rest of the interrupt frame to the stack.
        PUSH_IFRAME_BOTTOM(IFRAME_TYPE_OTHER)
        cld

        // Frame pointer is the iframe.
        movq    %rsp, %rbp

        // Set the RF (resume flag) in RFLAGS. This prevents an instruction
        // breakpoint on the instruction we're returning to to trigger a debug
        // exception.
        orq             $X86_EFLAGS_RESUME, IFRAME_flags(%rbp)

        // xsave needs a 64-byte alignment
        andq    $~63, %rsp
        movq    (gFPUSaveLength), %rcx
        subq    %rcx, %rsp
        leaq    (%rsp), %rdi
        shrq    $3, %rcx
        movq    $0, %rax
        rep stosq
        movl    (gXsaveMask), %eax
        movl    (gXsaveMask+4), %edx
        movq    %rsp, %rdi
        CODEPATCH_START
        fxsaveq (%rdi)
        CODEPATCH_END(ALTCODEPATCH_TAG_XSAVE)

        // Call the interrupt handler.
        movq    %rbp, %rdi
        movq    IFRAME_vector(%rbp), %rax
        call    *gInterruptHandlerTable(, %rax, 8)

        movl    (gXsaveMask), %eax
        movl    (gXsaveMask+4), %edx
        movq    %rsp, %rdi
        CODEPATCH_START
        fxrstorq        (%rdi)
        CODEPATCH_END(ALTCODEPATCH_TAG_XRSTOR)
        movq    %rbp, %rsp

        // Restore the saved registers.
        RESTORE_IFRAME()

        iretq
FUNCTION_END(intr_bottom)


// Handler for an interrupt that occurred in user-mode.
STATIC_FUNCTION(intr_bottom_user):
        // Load the kernel GS segment base.
        swapgs
        lfence

        // Push the rest of the interrupt frame to the stack.
        PUSH_IFRAME_BOTTOM(IFRAME_TYPE_OTHER)
        cld

        // Frame pointer is the iframe.
        movq    %rsp, %rbp
        andq    $~15, %rsp

        // Get thread pointer.
        movq    %gs:0, %r12

        movl    (gXsaveMask), %eax
        movl    (gXsaveMask+4), %edx
        leaq    THREAD_user_fpu_state(%r12), %rdi
        CODEPATCH_START
        fxsaveq (%rdi)
        CODEPATCH_END(ALTCODEPATCH_TAG_XSAVE)
        movq    %rdi, IFRAME_fpu(%rbp)

        // Set the RF (resume flag) in RFLAGS. This prevents an instruction
        // breakpoint on the instruction we're returning to to trigger a debug
        // exception.
        orq             $X86_EFLAGS_RESUME, IFRAME_flags(%rbp)

        STOP_USER_DEBUGGING()
        UPDATE_THREAD_USER_TIME()

        // Call the interrupt handler.
        movq    %rbp, %rdi
        movq    IFRAME_vector(%rbp), %rax
        call    *gInterruptHandlerTable(, %rax, 8)

        // If there are no signals pending or we're not debugging, we can avoid
        // most of the work here, just need to update the kernel time.
        testl   $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \
                        | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \
                        | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
                        , THREAD_flags(%r12)
        jnz             .Lkernel_exit_work

        cli

        UPDATE_THREAD_KERNEL_TIME()

        movl    (gXsaveMask), %eax
        movl    (gXsaveMask+4), %edx
        leaq    THREAD_user_fpu_state(%r12), %rdi
        CODEPATCH_START
        fxrstorq        (%rdi)
        CODEPATCH_END(ALTCODEPATCH_TAG_XRSTOR)
        movq    %rbp, %rsp

        // Restore the saved registers.
        RESTORE_IFRAME()

        // Restore the previous GS base and return.
        swapgs
        iretq

.Lkernel_exit_work:
        // Slow path for return to userland.

        // Do we need to handle signals?
        testl   $(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD \
                        | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
                        , THREAD_flags(%r12)
        jnz             .Lkernel_exit_handle_signals
        cli
        call    thread_at_kernel_exit_no_signals

.Lkernel_exit_work_done:
        // Install breakpoints, if defined.
        testl   $THREAD_FLAGS_BREAKPOINTS_DEFINED, THREAD_flags(%r12)
        jz              1f
        movq    %rbp, %rdi
        call    x86_init_user_debug_at_kernel_exit
1:
        movl    (gXsaveMask), %eax
        movl    (gXsaveMask+4), %edx
        leaq    THREAD_user_fpu_state(%r12), %rdi
        CODEPATCH_START
        fxrstorq        (%rdi)
        CODEPATCH_END(ALTCODEPATCH_TAG_XRSTOR)
        movq    %rbp, %rsp

        // Restore the saved registers.
        RESTORE_IFRAME()

        // Restore the previous GS base and return.
        swapgs
        iretq

.Lkernel_exit_handle_signals:
        // thread_at_kernel_exit requires interrupts to be enabled, it will disable
        // them after.
        sti
        call    thread_at_kernel_exit
        jmp             .Lkernel_exit_work_done
FUNCTION_END(intr_bottom_user)


// SYSCALL entry point.
FUNCTION(x86_64_syscall_entry):
        // Upon entry, RSP still points at the user stack.  Load the kernel GS
        // segment base address, which points at the current thread's arch_thread
        // structure. This contains our kernel stack pointer and a temporary
        // scratch space to store the user stack pointer in before we can push it
        // to the stack.
        swapgs
        movq    %rsp, %gs:ARCH_THREAD_user_rsp
        movq    %gs:ARCH_THREAD_syscall_rsp, %rsp

        // The following pushes de-align the stack by 8 bytes, so account for that first.
        sub     $8, %rsp

        // Set up an iframe on the stack (R11 = saved RFLAGS, RCX = saved RIP).
        push    $USER_DATA_SELECTOR                     // ss
        push    %gs:ARCH_THREAD_user_rsp        // rsp
        push    %r11                                            // flags
        push    $USER_CODE_SELECTOR                     // cs
        push    %rcx                                            // ip
        push    $0                                                      // error_code
        push    $99                                                     // vector
        PUSH_IFRAME_BOTTOM(IFRAME_TYPE_SYSCALL)
        cld

        // Frame pointer is the iframe.
        movq    %rsp, %rbp

        // Preserve call number (R14 is callee-save), get thread pointer.
        movq    %rax, %r14
        movq    %gs:0, %r12

        STOP_USER_DEBUGGING()
        UPDATE_THREAD_USER_TIME()

        // No longer need interrupts disabled.
        sti

        // Check whether the syscall number is valid.
        cmpq    $SYSCALL_COUNT, %r14
        jae             .Lsyscall_return

        // Get the system call table entry. Note I'm hardcoding the shift because
        // sizeof(syscall_info) is 16 and scale factors of 16 aren't supported,
        // so can't just do leaq kSyscallInfos(, %rax, SYSCALL_INFO_sizeof).
        movq    %r14, %rax
        shlq    $4, %rax
        leaq    kSyscallInfos(, %rax, 1), %rax

        // Check the number of call arguments, greater than 6 (6 * 8 = 48) requires
        // a stack copy.
        movq    SYSCALL_INFO_parameter_size(%rax), %rcx
        cmpq    $48, %rcx
        ja              .Lsyscall_stack_args

.Lperform_syscall:
        testl   $THREAD_FLAGS_DEBUGGER_INSTALLED, THREAD_flags(%r12)
        jnz             .Lpre_syscall_debug

.Lpre_syscall_debug_done:
        // Restore the arguments from the iframe. UPDATE_THREAD_USER_TIME() makes
        // 2 function calls which means they may have been overwritten. Note that
        // argument 4 is in R10 on the frame rather than RCX as RCX is used by
        // SYSCALL.
        movq    IFRAME_di(%rbp), %rdi
        movq    IFRAME_si(%rbp), %rsi
        movq    IFRAME_dx(%rbp), %rdx
        movq    IFRAME_r10(%rbp), %rcx
        movq    IFRAME_r8(%rbp), %r8
        movq    IFRAME_r9(%rbp), %r9

        // TODO: pre-syscall tracing

        // Call the function and save its return value.
        call    *SYSCALL_INFO_function(%rax)
        movq    %rax, IFRAME_ax(%rbp)

        // TODO: post-syscall tracing

.Lsyscall_return:
        // Restore the original stack pointer and return.
        movq    %rbp, %rsp

        // Clear the restarted flag.
        testl   $THREAD_FLAGS_SYSCALL_RESTARTED, THREAD_flags(%r12)
        jz              2f
1:
        movl    THREAD_flags(%r12), %eax
        movl    %eax, %edx
        andl    $~THREAD_FLAGS_SYSCALL_RESTARTED, %edx
        lock
        cmpxchgl        %edx, THREAD_flags(%r12)
        jnz             1b
2:
        testl   $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \
                        | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \
                        | THREAD_FLAGS_TRAP_FOR_CORE_DUMP | THREAD_FLAGS_RESTART_SYSCALL) \
                        , THREAD_flags(%r12)
        jnz             .Lpost_syscall_work

        cli

        UPDATE_THREAD_KERNEL_TIME()

        // If we've just restored a signal frame, use the IRET path.
        cmpq    $SYSCALL_RESTORE_SIGNAL_FRAME, %r14
        je              .Lrestore_fpu

        CLEAR_FPU_STATE()

        // Restore the iframe and RCX/R11 for SYSRET.
        RESTORE_IFRAME()
        pop             %rcx
        addq    $8, %rsp
        pop             %r11
        pop             %rsp

        // Restore previous GS base and return.
        swapgs
        sysretq

.Lpre_syscall_debug:
        // user_debug_pre_syscall expects a pointer to a block of arguments, need
        // to push the register arguments onto the stack.
        push    IFRAME_r9(%rbp)
        push    IFRAME_r8(%rbp)
        push    IFRAME_r10(%rbp)
        push    IFRAME_dx(%rbp)
        push    IFRAME_si(%rbp)
        push    IFRAME_di(%rbp)
        movq    %r14, %rdi                              // syscall number
        movq    %rsp, %rsi
        subq    $8, %rsp
        push    %rax
        call    user_debug_pre_syscall
        pop             %rax
        addq    $56, %rsp
        jmp             .Lpre_syscall_debug_done

.Lpost_syscall_work:
        testl   $THREAD_FLAGS_DEBUGGER_INSTALLED, THREAD_flags(%r12)
        jz              1f

        // Post-syscall debugging. Same as above, need a block of arguments.
        push    IFRAME_r9(%rbp)
        push    IFRAME_r8(%rbp)
        push    IFRAME_r10(%rbp)
        push    IFRAME_dx(%rbp)
        push    IFRAME_si(%rbp)
        push    IFRAME_di(%rbp)
        movq    %r14, %rdi                              // syscall number
        movq    %rsp, %rsi
        movq    IFRAME_ax(%rbp), %rdx   // return value
        call    user_debug_post_syscall
        addq    $48, %rsp
1:
        // Do we need to handle signals?
        testl   $(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD \
                        | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
                        , THREAD_flags(%r12)
        jnz             .Lpost_syscall_handle_signals
        cli
        call    thread_at_kernel_exit_no_signals

.Lpost_syscall_work_done:
        // Handle syscall restarting.
        testl   $THREAD_FLAGS_RESTART_SYSCALL, THREAD_flags(%r12)
        jz              1f
        movq    %rsp, %rdi
        call    x86_restart_syscall
1:
        // Install breakpoints, if defined.
        testl   $THREAD_FLAGS_BREAKPOINTS_DEFINED, THREAD_flags(%r12)
        jz              1f
        movq    %rbp, %rdi
        call    x86_init_user_debug_at_kernel_exit
1:
        // On this return path it is possible that the frame has been modified,
        // for example to execute a signal handler. In this case it is safer to
        // return via IRET.
        CLEAR_FPU_STATE()
        jmp .Liret

.Lrestore_fpu:
        movq    IFRAME_fpu(%rbp), %rdi

        movl    (gXsaveMask), %eax
        movl    (gXsaveMask+4), %edx
        CODEPATCH_START
        fxrstorq        (%rdi)
        CODEPATCH_END(ALTCODEPATCH_TAG_XRSTOR)
.Liret:
        // Restore the saved registers.
        RESTORE_IFRAME()

        // Restore the previous GS base and return.
        swapgs
        iretq

.Lpost_syscall_handle_signals:
        call    thread_at_kernel_exit
        jmp             .Lpost_syscall_work_done

.Lsyscall_stack_args:
        // Some arguments are on the stack, work out what we need to copy. 6
        // arguments (48 bytes) are already in registers.
        // RAX = syscall table entry address, RCX = argument size.
        subq    $48, %rcx

        // Get the address to copy from.
        movq    IFRAME_user_sp(%rbp), %rsi
        addq    $8, %rsi
        movabs  $(USER_BASE + USER_SIZE), %rdx
        cmp             %rdx, %rsi
        jae             .Lbad_syscall_args

        // Make space on the stack.
        subq    %rcx, %rsp
        andq    $~15, %rsp
        movq    %rsp, %rdi

        // Set a fault handler.
        movq    $.Lbad_syscall_args, THREAD_fault_handler(%r12)

        ASM_STAC

        // Copy them by quadwords.
        shrq    $3, %rcx
        rep
        movsq
        ASM_CLAC
        movq    $0, THREAD_fault_handler(%r12)

        // Perform the call.
        jmp             .Lperform_syscall

.Lbad_syscall_args:
        movq    $0, THREAD_fault_handler(%r12)
        movq    %rbp, %rsp
        jmp             .Lsyscall_return
FUNCTION_END(x86_64_syscall_entry)


/*!     \fn void x86_return_to_userland(iframe* frame)
        \brief Returns to the userland environment given by \a frame.

        Before returning to userland all potentially necessary kernel exit work is
        done.

        \a frame must point to a location somewhere on the caller's stack (e.g. a
        local variable).
        The function must be called with interrupts disabled.

        \param frame The iframe defining the userland environment.
*/
FUNCTION(x86_return_to_userland):
        movq    %rdi, %rbp
        movq    %rbp, %rsp

        // Perform kernel exit work.
        movq    %gs:0, %r12
        testl   $(THREAD_FLAGS_DEBUGGER_INSTALLED | THREAD_FLAGS_SIGNALS_PENDING \
                        | THREAD_FLAGS_DEBUG_THREAD | THREAD_FLAGS_BREAKPOINTS_DEFINED \
                        | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
                        , THREAD_flags(%r12)
        jnz             .Luserland_return_work

        UPDATE_THREAD_KERNEL_TIME()

        CLEAR_FPU_STATE()

        // Restore the frame and return.
        RESTORE_IFRAME()
        swapgs
        iretq
.Luserland_return_work:
        // Slow path for return to userland.

        // Do we need to handle signals?
        testl   $(THREAD_FLAGS_SIGNALS_PENDING | THREAD_FLAGS_DEBUG_THREAD \
                        | THREAD_FLAGS_TRAP_FOR_CORE_DUMP) \
                        , THREAD_flags(%r12)
        jnz             .Luserland_return_handle_signals
        cli
        call    thread_at_kernel_exit_no_signals

.Luserland_return_work_done:
        // Install breakpoints, if defined.
        testl   $THREAD_FLAGS_BREAKPOINTS_DEFINED, THREAD_flags(%r12)
        jz              1f
        movq    %rbp, %rdi
        call    x86_init_user_debug_at_kernel_exit
1:
        CLEAR_FPU_STATE()

        // Restore the frame and return.
        RESTORE_IFRAME()
        swapgs
        iretq
.Luserland_return_handle_signals:
        // thread_at_kernel_exit requires interrupts to be enabled, it will disable
        // them after.
        sti
        call    thread_at_kernel_exit
        jmp             .Luserland_return_work_done
FUNCTION_END(x86_return_to_userland)