/*
 * This file is subject to the terms and conditions of the GNU General Public
 * License.  See the file "COPYING" in the main directory of this archive
 * for more details.
 *
 * Copyright (C) 1994 - 2000, 2001, 2003 Ralf Baechle
 * Copyright (C) 1999, 2000 Silicon Graphics, Inc.
 * Copyright (C) 2002, 2007  Maciej W. Rozycki
 * Copyright (C) 2001, 2012 MIPS Technologies, Inc.  All rights reserved.
 */
#include <linux/init.h>

#include <asm/asm.h>
#include <asm/asmmacro.h>
#include <asm/cacheops.h>
#include <asm/irqflags.h>
#include <asm/regdef.h>
#include <asm/fpregdef.h>
#include <asm/mipsregs.h>
#include <asm/stackframe.h>
#include <asm/sync.h>
#include <asm/thread_info.h>

        __INIT

/*
 * General exception vector for all other CPUs.
 *
 * Be careful when changing this, it has to be at most 128 bytes
 * to fit into space reserved for the exception handler.
 */
NESTED(except_vec3_generic, 0, sp)
        .set    push
        .set    noat
        mfc0    k1, CP0_CAUSE
        andi    k1, k1, 0x7c
#ifdef CONFIG_64BIT
        dsll    k1, k1, 1
#endif
        PTR_L   k0, exception_handlers(k1)
        jr      k0
        .set    pop
        END(except_vec3_generic)

/*
 * General exception handler for CPUs with virtual coherency exception.
 *
 * Be careful when changing this, it has to be at most 256 (as a special
 * exception) bytes to fit into space reserved for the exception handler.
 */
NESTED(except_vec3_r4000, 0, sp)
        .set    push
        .set    arch=r4000
        .set    noat
        mfc0    k1, CP0_CAUSE
        li      k0, 31<<2
        andi    k1, k1, 0x7c
        .set    push
        .set    noreorder
        .set    nomacro
        beq     k1, k0, handle_vced
         li     k0, 14<<2
        beq     k1, k0, handle_vcei
#ifdef CONFIG_64BIT
         dsll   k1, k1, 1
#endif
        .set    pop
        PTR_L   k0, exception_handlers(k1)
        jr      k0

        /*
         * Big shit, we now may have two dirty primary cache lines for the same
         * physical address.  We can safely invalidate the line pointed to by
         * c0_badvaddr because after return from this exception handler the
         * load / store will be re-executed.
         */
handle_vced:
        MFC0    k0, CP0_BADVADDR
        li      k1, -4                                  # Is this ...
        and     k0, k1                                  # ... really needed?
        mtc0    zero, CP0_TAGLO
        cache   Index_Store_Tag_D, (k0)
        cache   Hit_Writeback_Inv_SD, (k0)
#ifdef CONFIG_PROC_FS
        PTR_LA  k0, vced_count
        lw      k1, (k0)
        addiu   k1, 1
        sw      k1, (k0)
#endif
        eret

handle_vcei:
        MFC0    k0, CP0_BADVADDR
        cache   Hit_Writeback_Inv_SD, (k0)              # also cleans pi
#ifdef CONFIG_PROC_FS
        PTR_LA  k0, vcei_count
        lw      k1, (k0)
        addiu   k1, 1
        sw      k1, (k0)
#endif
        eret
        .set    pop
        END(except_vec3_r4000)

        __FINIT

        .section .cpuidle.text,"ax"
        /* Align to 32 bytes for the maximum idle interrupt region size. */
        .align  5
LEAF(r4k_wait)
        /* Keep the ISA bit clear for calculations on local labels here. */
0:      .fill   0
        /* Start of idle interrupt region. */
        local_irq_enable
        /*
         * If an interrupt lands here, before going idle on the next
         * instruction, we must *NOT* go idle since the interrupt could
         * have set TIF_NEED_RESCHED or caused a timer to need resched.
         * Fall through -- see skipover_handler below -- and have the
         * idle loop take care of things.
         */
1:      .fill   0
        /* The R2 EI/EHB sequence takes 8 bytes, otherwise pad up.  */
        .if     1b - 0b > 32
        .error  "overlong idle interrupt region"
        .elseif 1b - 0b > 8
        .align  4
        .endif
2:      .fill   0
        .equ    r4k_wait_idle_size, 2b - 0b
        /* End of idle interrupt region; size has to be a power of 2. */
        .set    MIPS_ISA_ARCH_LEVEL_RAW
r4k_wait_insn:
        wait
r4k_wait_exit:
        .set    mips0
        local_irq_disable
        jr      ra
        END(r4k_wait)
        .previous

        .macro  BUILD_SKIPOVER_PROLOGUE handler
        FEXPORT(skipover_\handler)
        .set    push
        .set    noat
        MFC0    k0, CP0_EPC
        /* Subtract/add 2 to let the ISA bit propagate through the mask.  */
        PTR_LA  k1, r4k_wait_insn - 2
        ori     k0, r4k_wait_idle_size - 2
        .set    noreorder
        bne     k0, k1, \handler
         PTR_ADDIU      k0, r4k_wait_exit - r4k_wait_insn + 2
        .set    reorder
        MTC0    k0, CP0_EPC
        .set pop
        .endm

        .align  5
BUILD_SKIPOVER_PROLOGUE handle_int
NESTED(handle_int, PT_SIZE, sp)
        .cfi_signal_frame
#ifdef CONFIG_TRACE_IRQFLAGS
        /*
         * Check to see if the interrupted code has just disabled
         * interrupts and ignore this interrupt for now if so.
         *
         * local_irq_disable() disables interrupts and then calls
         * trace_hardirqs_off() to track the state. If an interrupt is taken
         * after interrupts are disabled but before the state is updated
         * it will appear to restore_all that it is incorrectly returning with
         * interrupts disabled
         */
        .set    push
        .set    noat
        mfc0    k0, CP0_STATUS
#if defined(CONFIG_CPU_R3000)
        and     k0, ST0_IEP
        bnez    k0, 1f

        mfc0    k0, CP0_EPC
        .set    noreorder
        j       k0
         rfe
#else
        and     k0, ST0_IE
        bnez    k0, 1f

        eret
#endif
1:
        .set pop
#endif
        SAVE_ALL docfi=1
        CLI
        TRACE_IRQS_OFF

        LONG_L  s0, TI_REGS($28)
        LONG_S  sp, TI_REGS($28)

        /*
         * SAVE_ALL ensures we are using a valid kernel stack for the thread.
         * Check if we are already using the IRQ stack.
         */
        move    s1, sp # Preserve the sp

        /* Get IRQ stack for this CPU */
        ASM_CPUID_MFC0  k0, ASM_SMP_CPUID_REG
#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32)
        lui     k1, %hi(irq_stack)
#else
        lui     k1, %highest(irq_stack)
        daddiu  k1, %higher(irq_stack)
        dsll    k1, 16
        daddiu  k1, %hi(irq_stack)
        dsll    k1, 16
#endif
        LONG_SRL        k0, SMP_CPUID_PTRSHIFT
        LONG_ADDU       k1, k0
        LONG_L  t0, %lo(irq_stack)(k1)

        # Check if already on IRQ stack
        PTR_LI  t1, ~(_THREAD_SIZE-1)
        and     t1, t1, sp
        beq     t0, t1, 2f

        /* Switch to IRQ stack */
        li      t1, _IRQ_STACK_START
        PTR_ADD sp, t0, t1

        /* Save task's sp on IRQ stack so that unwinding can follow it */
        LONG_S  s1, 0(sp)
2:
        jal     plat_irq_dispatch

        /* Restore sp */
        move    sp, s1

        j       ret_from_irq
#ifdef CONFIG_CPU_MICROMIPS
        nop
#endif
        END(handle_int)

        __INIT

/*
 * Special interrupt vector for MIPS64 ISA & embedded MIPS processors.
 * This is a dedicated interrupt exception vector which reduces the
 * interrupt processing overhead.  The jump instruction will be replaced
 * at the initialization time.
 *
 * Be careful when changing this, it has to be at most 128 bytes
 * to fit into space reserved for the exception handler.
 */
NESTED(except_vec4, 0, sp)
1:      j       1b                      /* Dummy, will be replaced */
        END(except_vec4)

/*
 * EJTAG debug exception handler.
 * The EJTAG debug exception entry point is 0xbfc00480, which
 * normally is in the boot PROM, so the boot PROM must do an
 * unconditional jump to this vector.
 */
NESTED(except_vec_ejtag_debug, 0, sp)
        j       ejtag_debug_handler
#ifdef CONFIG_CPU_MICROMIPS
         nop
#endif
        END(except_vec_ejtag_debug)

        __FINIT

/*
 * Vectored interrupt handler.
 * This prototype is copied to ebase + n*IntCtl.VS and patched
 * to invoke the handler
 */
BUILD_SKIPOVER_PROLOGUE except_vec_vi
NESTED(except_vec_vi, 0, sp)
        SAVE_SOME docfi=1
        SAVE_AT docfi=1
        .set    push
        .set    noreorder
        PTR_LA  v1, except_vec_vi_handler
        jr      v1
FEXPORT(except_vec_vi_ori)
         ori    v0, zero, 0             /* Offset in vi_handlers[] */
        .set    pop
        END(except_vec_vi)
EXPORT(except_vec_vi_end)

/*
 * Common Vectored Interrupt code
 * Complete the register saves and invoke the handler, $v0 holds
 * offset into vi_handlers[]
 */
NESTED(except_vec_vi_handler, 0, sp)
        SAVE_TEMP
        SAVE_STATIC
        CLI
#ifdef CONFIG_TRACE_IRQFLAGS
        move    s0, v0
        TRACE_IRQS_OFF
        move    v0, s0
#endif

        LONG_L  s0, TI_REGS($28)
        LONG_S  sp, TI_REGS($28)

        /*
         * SAVE_ALL ensures we are using a valid kernel stack for the thread.
         * Check if we are already using the IRQ stack.
         */
        move    s1, sp # Preserve the sp

        /* Get IRQ stack for this CPU */
        ASM_CPUID_MFC0  k0, ASM_SMP_CPUID_REG
#if defined(CONFIG_32BIT) || defined(KBUILD_64BIT_SYM32)
        lui     k1, %hi(irq_stack)
#else
        lui     k1, %highest(irq_stack)
        daddiu  k1, %higher(irq_stack)
        dsll    k1, 16
        daddiu  k1, %hi(irq_stack)
        dsll    k1, 16
#endif
        LONG_SRL        k0, SMP_CPUID_PTRSHIFT
        LONG_ADDU       k1, k0
        LONG_L  t0, %lo(irq_stack)(k1)

        # Check if already on IRQ stack
        PTR_LI  t1, ~(_THREAD_SIZE-1)
        and     t1, t1, sp
        beq     t0, t1, 2f

        /* Switch to IRQ stack */
        li      t1, _IRQ_STACK_START
        PTR_ADD sp, t0, t1

        /* Save task's sp on IRQ stack so that unwinding can follow it */
        LONG_S  s1, 0(sp)
2:
        PTR_L   v0, vi_handlers(v0)
        jalr    v0

        /* Restore sp */
        move    sp, s1

        j       ret_from_irq
        END(except_vec_vi_handler)

/*
 * EJTAG debug exception handler.
 */
NESTED(ejtag_debug_handler, PT_SIZE, sp)
        .set    push
        .set    noat
        MTC0    k0, CP0_DESAVE
        mfc0    k0, CP0_DEBUG

        andi    k0, k0, MIPS_DEBUG_DBP  # Check for SDBBP.
        beqz    k0, ejtag_return

#ifdef CONFIG_SMP
1:      PTR_LA  k0, ejtag_debug_buffer_spinlock
        __SYNC(full, loongson3_war)
2:      ll      k0, 0(k0)
        bnez    k0, 2b
        PTR_LA  k0, ejtag_debug_buffer_spinlock
        sc      k0, 0(k0)
        beqz    k0, 1b
# ifdef CONFIG_WEAK_REORDERING_BEYOND_LLSC
        sync
# endif

        PTR_LA  k0, ejtag_debug_buffer
        LONG_S  k1, 0(k0)

        ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG
        PTR_SRL k1, SMP_CPUID_PTRSHIFT
        PTR_SLL k1, LONGLOG
        PTR_LA  k0, ejtag_debug_buffer_per_cpu
        PTR_ADDU k0, k1

        PTR_LA  k1, ejtag_debug_buffer
        LONG_L  k1, 0(k1)
        LONG_S  k1, 0(k0)

        PTR_LA  k0, ejtag_debug_buffer_spinlock
        sw      zero, 0(k0)
#else
        PTR_LA  k0, ejtag_debug_buffer
        LONG_S  k1, 0(k0)
#endif

        SAVE_ALL
        move    a0, sp
        jal     ejtag_exception_handler
        RESTORE_ALL

#ifdef CONFIG_SMP
        ASM_CPUID_MFC0 k1, ASM_SMP_CPUID_REG
        PTR_SRL k1, SMP_CPUID_PTRSHIFT
        PTR_SLL k1, LONGLOG
        PTR_LA  k0, ejtag_debug_buffer_per_cpu
        PTR_ADDU k0, k1
        LONG_L  k1, 0(k0)
#else
        PTR_LA  k0, ejtag_debug_buffer
        LONG_L  k1, 0(k0)
#endif

ejtag_return:
        back_to_back_c0_hazard
        MFC0    k0, CP0_DESAVE
        .set    mips32
        deret
        .set    pop
        END(ejtag_debug_handler)

/*
 * This buffer is reserved for the use of the EJTAG debug
 * handler.
 */
        .data
EXPORT(ejtag_debug_buffer)
        .fill   LONGSIZE
#ifdef CONFIG_SMP
EXPORT(ejtag_debug_buffer_spinlock)
        .fill   LONGSIZE
EXPORT(ejtag_debug_buffer_per_cpu)
        .fill   LONGSIZE * NR_CPUS
#endif
        .previous

        __INIT

/*
 * NMI debug exception handler for MIPS reference boards.
 * The NMI debug exception entry point is 0xbfc00000, which
 * normally is in the boot PROM, so the boot PROM must do a
 * unconditional jump to this vector.
 */
NESTED(except_vec_nmi, 0, sp)
        j       nmi_handler
#ifdef CONFIG_CPU_MICROMIPS
         nop
#endif
        END(except_vec_nmi)

        __FINIT

NESTED(nmi_handler, PT_SIZE, sp)
        .cfi_signal_frame
        .set    push
        .set    noat
        /*
         * Clear ERL - restore segment mapping
         * Clear BEV - required for page fault exception handler to work
         */
        mfc0    k0, CP0_STATUS
        ori     k0, k0, ST0_EXL
        li      k1, ~(ST0_BEV | ST0_ERL)
        and     k0, k0, k1
        mtc0    k0, CP0_STATUS
        _ehb
        SAVE_ALL
        move    a0, sp
        jal     nmi_exception_handler
        /* nmi_exception_handler never returns */
        .set    pop
        END(nmi_handler)

        .macro  __build_clear_none
        .endm

        .macro  __build_clear_sti
        TRACE_IRQS_ON
        STI
        .endm

        .macro  __build_clear_cli
        CLI
        TRACE_IRQS_OFF
        .endm

        .macro  __build_clear_fpe
        CLI
        TRACE_IRQS_OFF
        .set    push
        /* gas fails to assemble cfc1 for some archs (octeon).*/ \
        .set    mips1
        .set    hardfloat
        cfc1    a1, fcr31
        .set    pop
        .endm

        .macro  __build_clear_msa_fpe
        CLI
        TRACE_IRQS_OFF
        _cfcmsa a1, MSA_CSR
        .endm

        .macro  __build_clear_ade
        MFC0    t0, CP0_BADVADDR
        PTR_S   t0, PT_BVADDR(sp)
        KMODE
        .endm

        .macro __build_clear_gsexc
        .set    push
        /*
         * We need to specify a selector to access the CP0.Diag1 (GSCause)
         * register. All GSExc-equipped processors have MIPS32.
         */
        .set    mips32
        mfc0    a1, CP0_DIAGNOSTIC1
        .set    pop
        TRACE_IRQS_ON
        STI
        .endm

        .macro  __BUILD_silent exception
        .endm

        /* Gas tries to parse the ASM_PRINT argument as a string containing
           string escapes and emits bogus warnings if it believes to
           recognize an unknown escape code.  So make the arguments
           start with an n and gas will believe \n is ok ...  */
        .macro  __BUILD_verbose nexception
        LONG_L  a1, PT_EPC(sp)
#ifdef CONFIG_32BIT
        ASM_PRINT("Got \nexception at %08lx\012")
#endif
#ifdef CONFIG_64BIT
        ASM_PRINT("Got \nexception at %016lx\012")
#endif
        .endm

        .macro  __BUILD_count exception
        LONG_L  t0,exception_count_\exception
        LONG_ADDIU      t0, 1
        LONG_S  t0,exception_count_\exception
        .comm   exception_count\exception, 8, 8
        .endm

        .macro  __BUILD_HANDLER exception handler clear verbose ext
        .align  5
        NESTED(handle_\exception, PT_SIZE, sp)
        .cfi_signal_frame
        .set    noat
        SAVE_ALL
        FEXPORT(handle_\exception\ext)
        __build_clear_\clear
        .set    at
        __BUILD_\verbose \exception
        move    a0, sp
        jal     do_\handler
        j       ret_from_exception
        END(handle_\exception)
        .endm

        .macro  BUILD_HANDLER exception handler clear verbose
        __BUILD_HANDLER \exception \handler \clear \verbose _int
        .endm

        BUILD_HANDLER adel ade ade silent               /* #4  */
        BUILD_HANDLER ades ade ade silent               /* #5  */
        BUILD_HANDLER ibe be cli silent                 /* #6  */
        BUILD_HANDLER dbe be cli silent                 /* #7  */
        BUILD_HANDLER bp bp sti silent                  /* #9  */
        BUILD_HANDLER ri ri sti silent                  /* #10 */
        BUILD_HANDLER cpu cpu sti silent                /* #11 */
        BUILD_HANDLER ov ov sti silent                  /* #12 */
        BUILD_HANDLER tr tr sti silent                  /* #13 */
        BUILD_HANDLER msa_fpe msa_fpe msa_fpe silent    /* #14 */
#ifdef CONFIG_MIPS_FP_SUPPORT
        BUILD_HANDLER fpe fpe fpe silent                /* #15 */
#endif
        BUILD_HANDLER ftlb ftlb none silent             /* #16 */
        BUILD_HANDLER gsexc gsexc gsexc silent          /* #16 */
        BUILD_HANDLER msa msa sti silent                /* #21 */
        BUILD_HANDLER mdmx mdmx sti silent              /* #22 */
#ifdef  CONFIG_HARDWARE_WATCHPOINTS
        /*
         * For watch, interrupts will be enabled after the watch
         * registers are read.
         */
        BUILD_HANDLER watch watch cli silent            /* #23 */
#else
        BUILD_HANDLER watch watch sti verbose           /* #23 */
#endif
        BUILD_HANDLER mcheck mcheck cli verbose         /* #24 */
        BUILD_HANDLER mt mt sti silent                  /* #25 */
        BUILD_HANDLER dsp dsp sti silent                /* #26 */
        BUILD_HANDLER reserved reserved sti verbose     /* others */

        .align  5
        LEAF(handle_ri_rdhwr_tlbp)
        .set    push
        .set    noat
        .set    noreorder
        /* check if TLB contains a entry for EPC */
        MFC0    k1, CP0_ENTRYHI
        andi    k1, MIPS_ENTRYHI_ASID | MIPS_ENTRYHI_ASIDX
        MFC0    k0, CP0_EPC
        PTR_SRL k0, _PAGE_SHIFT + 1
        PTR_SLL k0, _PAGE_SHIFT + 1
        or      k1, k0
        MTC0    k1, CP0_ENTRYHI
        mtc0_tlbw_hazard
        tlbp
        tlb_probe_hazard
        mfc0    k1, CP0_INDEX
        .set    pop
        bltz    k1, handle_ri   /* slow path */
        /* fall thru */
        END(handle_ri_rdhwr_tlbp)

        LEAF(handle_ri_rdhwr)
        .set    push
        .set    noat
        .set    noreorder
        /* MIPS32:    0x7c03e83b: rdhwr v1,$29 */
        /* microMIPS: 0x007d6b3c: rdhwr v1,$29 */
        MFC0    k1, CP0_EPC
#if defined(CONFIG_CPU_MICROMIPS) || defined(CONFIG_CPU_MIPS32_R2) || defined(CONFIG_CPU_MIPS64_R2)
        and     k0, k1, 1
        beqz    k0, 1f
         xor    k1, k0
        lhu     k0, (k1)
        lhu     k1, 2(k1)
        ins     k1, k0, 16, 16
        lui     k0, 0x007d
        b       docheck
         ori    k0, 0x6b3c
1:
        lui     k0, 0x7c03
        lw      k1, (k1)
        ori     k0, 0xe83b
#else
        andi    k0, k1, 1
        bnez    k0, handle_ri
         lui    k0, 0x7c03
        lw      k1, (k1)
        ori     k0, 0xe83b
#endif
        .set    reorder
docheck:
        bne     k0, k1, handle_ri       /* if not ours */

isrdhwr:
        /* The insn is rdhwr.  No need to check CAUSE.BD here. */
        get_saved_sp    /* k1 := current_thread_info */
        .set    noreorder
        MFC0    k0, CP0_EPC
#if defined(CONFIG_CPU_R3000)
        ori     k1, _THREAD_MASK
        xori    k1, _THREAD_MASK
        LONG_L  v1, TI_TP_VALUE(k1)
        LONG_ADDIU      k0, 4
        jr      k0
         rfe
#else
#ifndef CONFIG_CPU_DADDI_WORKAROUNDS
        LONG_ADDIU      k0, 4           /* stall on $k0 */
#else
        .set    at=v1
        LONG_ADDIU      k0, 4
        .set    noat
#endif
        MTC0    k0, CP0_EPC
        /* I hope three instructions between MTC0 and ERET are enough... */
        ori     k1, _THREAD_MASK
        xori    k1, _THREAD_MASK
        LONG_L  v1, TI_TP_VALUE(k1)
        .set    push
        .set    arch=r4000
        eret
        .set    pop
#endif
        .set    pop
        END(handle_ri_rdhwr)

#ifdef CONFIG_CPU_R4X00_BUGS64
/* A temporary overflow handler used by check_daddi(). */

        __INIT

        BUILD_HANDLER  daddi_ov daddi_ov none silent    /* #12 */
#endif