/* SPDX-License-Identifier: GPL-2.0-or-later */
/*
 *  PowerPC version
 *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
 *  Rewritten by Cort Dougan (cort@fsmlabs.com) for PReP
 *    Copyright (C) 1996 Cort Dougan <cort@fsmlabs.com>
 *  Adapted for Power Macintosh by Paul Mackerras.
 *  Low-level exception handlers and MMU support
 *  rewritten by Paul Mackerras.
 *    Copyright (C) 1996 Paul Mackerras.
 *  MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
 *
 *  This file contains the system call entry code, context switch
 *  code, and exception/interrupt return code for PowerPC.
 */

#include <linux/errno.h>
#include <linux/err.h>
#include <linux/sys.h>
#include <linux/threads.h>
#include <linux/linkage.h>

#include <asm/reg.h>
#include <asm/page.h>
#include <asm/mmu.h>
#include <asm/cputable.h>
#include <asm/thread_info.h>
#include <asm/ppc_asm.h>
#include <asm/asm-offsets.h>
#include <asm/unistd.h>
#include <asm/ptrace.h>
#include <asm/feature-fixups.h>
#include <asm/barrier.h>
#include <asm/kup.h>
#include <asm/bug.h>
#include <asm/interrupt.h>

#include "head_32.h"

/*
 * powerpc relies on return from interrupt/syscall being context synchronising
 * (which rfi is) to support ARCH_HAS_MEMBARRIER_SYNC_CORE without additional
 * synchronisation instructions.
 */

/*
 * Align to 4k in order to ensure that all functions modyfing srr0/srr1
 * fit into one page in order to not encounter a TLB miss between the
 * modification of srr0/srr1 and the associated rfi.
 */
        .align  12

#if defined(CONFIG_PPC_BOOK3S_32) || defined(CONFIG_PPC_E500)
        .globl  prepare_transfer_to_handler
prepare_transfer_to_handler:
        /* if from kernel, check interrupted DOZE/NAP mode */
        lwz     r12,TI_LOCAL_FLAGS(r2)
        mtcrf   0x01,r12
        bt-     31-TLF_NAPPING,4f
        bt-     31-TLF_SLEEPING,7f
        blr

4:      rlwinm  r12,r12,0,~_TLF_NAPPING
        stw     r12,TI_LOCAL_FLAGS(r2)
        b       power_save_ppc32_restore

7:      rlwinm  r12,r12,0,~_TLF_SLEEPING
        stw     r12,TI_LOCAL_FLAGS(r2)
        lwz     r9,_MSR(r11)            /* if sleeping, clear MSR.EE */
        rlwinm  r9,r9,0,~MSR_EE
        lwz     r12,_LINK(r11)          /* and return to address in LR */
        REST_GPR(2, r11)
        b       fast_exception_return
_ASM_NOKPROBE_SYMBOL(prepare_transfer_to_handler)
#endif /* CONFIG_PPC_BOOK3S_32 || CONFIG_PPC_E500 */

#if defined(CONFIG_PPC_KUEP) && defined(CONFIG_PPC_BOOK3S_32)
SYM_FUNC_START(__kuep_lock)
        lwz     r9, THREAD+THSR0(r2)
        update_user_segments_by_4 r9, r10, r11, r12
        blr
SYM_FUNC_END(__kuep_lock)

SYM_FUNC_START_LOCAL(__kuep_unlock)
        lwz     r9, THREAD+THSR0(r2)
        rlwinm  r9,r9,0,~SR_NX
        update_user_segments_by_4 r9, r10, r11, r12
        blr
SYM_FUNC_END(__kuep_unlock)

.macro  kuep_lock
        bl      __kuep_lock
.endm
.macro  kuep_unlock
        bl      __kuep_unlock
.endm
#else
.macro  kuep_lock
.endm
.macro  kuep_unlock
.endm
#endif

        .globl  transfer_to_syscall
transfer_to_syscall:
        stw     r3, ORIG_GPR3(r1)
        stw     r11, GPR1(r1)
        stw     r11, 0(r1)
        mflr    r12
        stw     r12, _LINK(r1)
#ifdef CONFIG_BOOKE
        rlwinm  r9,r9,0,14,12           /* clear MSR_WE (necessary?) */
#endif
        lis     r12,STACK_FRAME_REGS_MARKER@ha /* exception frame marker */
        SAVE_GPR(2, r1)
        addi    r12,r12,STACK_FRAME_REGS_MARKER@l
        stw     r9,_MSR(r1)
        li      r2, INTERRUPT_SYSCALL
        stw     r12,STACK_INT_FRAME_MARKER(r1)
        stw     r2,_TRAP(r1)
        SAVE_GPR(0, r1)
        SAVE_GPRS(3, 8, r1)
        addi    r2,r10,-THREAD
        SAVE_NVGPRS(r1)
        kuep_lock

        /* Calling convention has r3 = regs, r4 = orig r0 */
        addi    r3,r1,STACK_INT_FRAME_REGS
        mr      r4,r0
        bl      system_call_exception

ret_from_syscall:
        addi    r4,r1,STACK_INT_FRAME_REGS
        li      r5,0
        bl      syscall_exit_prepare
#ifdef CONFIG_PPC_47x
        lis     r4,icache_44x_need_flush@ha
        lwz     r5,icache_44x_need_flush@l(r4)
        cmplwi  cr0,r5,0
        bne-    .L44x_icache_flush
#endif /* CONFIG_PPC_47x */
.L44x_icache_flush_return:
        kuep_unlock
        lwz     r4,_LINK(r1)
        lwz     r5,_CCR(r1)
        mtlr    r4
        lwz     r7,_NIP(r1)
        lwz     r8,_MSR(r1)
        cmpwi   r3,0
        REST_GPR(3, r1)
syscall_exit_finish:
        mtspr   SPRN_SRR0,r7
        mtspr   SPRN_SRR1,r8

        bne     3f
        mtcr    r5

1:      REST_GPR(2, r1)
        REST_GPR(1, r1)
        rfi

3:      mtcr    r5
        lwz     r4,_CTR(r1)
        lwz     r5,_XER(r1)
        REST_NVGPRS(r1)
        mtctr   r4
        mtxer   r5
        REST_GPR(0, r1)
        REST_GPRS(3, 12, r1)
        b       1b
_ASM_NOKPROBE_SYMBOL(syscall_exit_finish)

#ifdef CONFIG_44x
.L44x_icache_flush:
        li      r7,0
        iccci   r0,r0
        stw     r7,icache_44x_need_flush@l(r4)
        b       .L44x_icache_flush_return
#endif  /* CONFIG_44x */

        .globl  ret_from_fork
ret_from_fork:
        REST_NVGPRS(r1)
        bl      schedule_tail
        li      r3,0    /* fork() return value */
        b       ret_from_syscall

        .globl  ret_from_kernel_user_thread
ret_from_kernel_user_thread:
        bl      schedule_tail
        mtctr   r14
        mr      r3,r15
        PPC440EP_ERR42
        bctrl
        li      r3,0
        b       ret_from_syscall

        .globl  start_kernel_thread
start_kernel_thread:
        bl      schedule_tail
        mtctr   r14
        mr      r3,r15
        PPC440EP_ERR42
        bctrl
        /*
         * This must not return. We actually want to BUG here, not WARN,
         * because BUG will exit the process which is what the kernel thread
         * should have done, which may give some hope of continuing.
         */
100:    trap
        EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0

        .globl  fast_exception_return
fast_exception_return:
#ifndef CONFIG_BOOKE
        andi.   r10,r9,MSR_RI           /* check for recoverable interrupt */
        beq     3f                      /* if not, we've got problems */
#endif

        lwz     r10,_CCR(r11)
        REST_GPRS(1, 6, r11)
        mtcr    r10
        lwz     r10,_LINK(r11)
        mtlr    r10
        /* Clear the exception marker on the stack to avoid confusing stacktrace */
        li      r10, 0
        stw     r10, 8(r11)
        mtspr   SPRN_SRR1,r9
        mtspr   SPRN_SRR0,r12
        REST_GPR(9, r11)
        REST_GPR(10, r11)
        REST_GPR(12, r11)
        REST_GPR(11, r11)
        rfi
_ASM_NOKPROBE_SYMBOL(fast_exception_return)

/* aargh, a nonrecoverable interrupt, panic */
/* aargh, we don't know which trap this is */
3:
        li      r10,-1
        stw     r10,_TRAP(r11)
        prepare_transfer_to_handler
        bl      unrecoverable_exception
        trap    /* should not get here */

        .globl interrupt_return
interrupt_return:
        lwz     r4,_MSR(r1)
        addi    r3,r1,STACK_INT_FRAME_REGS
        andi.   r0,r4,MSR_PR
        beq     .Lkernel_interrupt_return
        bl      interrupt_exit_user_prepare
        cmpwi   r3,0
        kuep_unlock
        bne-    .Lrestore_nvgprs

.Lfast_user_interrupt_return:
        lwz     r11,_NIP(r1)
        lwz     r12,_MSR(r1)
        mtspr   SPRN_SRR0,r11
        mtspr   SPRN_SRR1,r12

BEGIN_FTR_SECTION
        lwarx   r0,0,r1
END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
        stwcx.  r0,0,r1         /* to clear the reservation */

        lwz     r3,_CCR(r1)
        lwz     r4,_LINK(r1)
        lwz     r5,_CTR(r1)
        lwz     r6,_XER(r1)
        li      r0,0

        /*
         * Leaving a stale exception marker on the stack can confuse
         * the reliable stack unwinder later on. Clear it.
         */
        stw     r0,8(r1)
        REST_GPRS(7, 12, r1)

        mtcr    r3
        mtlr    r4
        mtctr   r5
        mtspr   SPRN_XER,r6

        REST_GPRS(2, 6, r1)
        REST_GPR(0, r1)
        REST_GPR(1, r1)
        rfi

.Lrestore_nvgprs:
        REST_NVGPRS(r1)
        b       .Lfast_user_interrupt_return

.Lkernel_interrupt_return:
        bl      interrupt_exit_kernel_prepare

.Lfast_kernel_interrupt_return:
        cmpwi   cr1,r3,0
        lwz     r11,_NIP(r1)
        lwz     r12,_MSR(r1)
        mtspr   SPRN_SRR0,r11
        mtspr   SPRN_SRR1,r12

BEGIN_FTR_SECTION
        lwarx   r0,0,r1
END_FTR_SECTION_IFSET(CPU_FTR_NEED_PAIRED_STWCX)
        stwcx.  r0,0,r1         /* to clear the reservation */

        lwz     r3,_LINK(r1)
        lwz     r4,_CTR(r1)
        lwz     r5,_XER(r1)
        lwz     r6,_CCR(r1)
        li      r0,0

        REST_GPRS(7, 12, r1)

        mtlr    r3
        mtctr   r4
        mtspr   SPRN_XER,r5

        /*
         * Leaving a stale exception marker on the stack can confuse
         * the reliable stack unwinder later on. Clear it.
         */
        stw     r0,8(r1)

        REST_GPRS(2, 5, r1)

        bne-    cr1,1f /* emulate stack store */
        mtcr    r6
        REST_GPR(6, r1)
        REST_GPR(0, r1)
        REST_GPR(1, r1)
        rfi

1:      /*
         * Emulate stack store with update. New r1 value was already calculated
         * and updated in our interrupt regs by emulate_loadstore, but we can't
         * store the previous value of r1 to the stack before re-loading our
         * registers from it, otherwise they could be clobbered.  Use
         * SPRG Scratch0 as temporary storage to hold the store
         * data, as interrupts are disabled here so it won't be clobbered.
         */
        mtcr    r6
#ifdef CONFIG_BOOKE
        mtspr   SPRN_SPRG_WSCRATCH0, r9
#else
        mtspr   SPRN_SPRG_SCRATCH0, r9
#endif
        addi    r9,r1,INT_FRAME_SIZE /* get original r1 */
        REST_GPR(6, r1)
        REST_GPR(0, r1)
        REST_GPR(1, r1)
        stw     r9,0(r1) /* perform store component of stwu */
#ifdef CONFIG_BOOKE
        mfspr   r9, SPRN_SPRG_RSCRATCH0
#else
        mfspr   r9, SPRN_SPRG_SCRATCH0
#endif
        rfi
_ASM_NOKPROBE_SYMBOL(interrupt_return)

#ifdef CONFIG_BOOKE

/*
 * Returning from a critical interrupt in user mode doesn't need
 * to be any different from a normal exception.  For a critical
 * interrupt in the kernel, we just return (without checking for
 * preemption) since the interrupt may have happened at some crucial
 * place (e.g. inside the TLB miss handler), and because we will be
 * running with r1 pointing into critical_stack, not the current
 * process's kernel stack (and therefore current_thread_info() will
 * give the wrong answer).
 * We have to restore various SPRs that may have been in use at the
 * time of the critical interrupt.
 *
 */

#define RET_FROM_EXC_LEVEL(exc_lvl_srr0, exc_lvl_srr1, exc_lvl_rfi)     \
        REST_NVGPRS(r1);                                                \
        lwz     r3,_MSR(r1);                                            \
        andi.   r3,r3,MSR_PR;                                           \
        bne     interrupt_return;                                       \
        REST_GPR(0, r1);                                                \
        REST_GPRS(2, 8, r1);                                            \
        lwz     r10,_XER(r1);                                           \
        lwz     r11,_CTR(r1);                                           \
        mtspr   SPRN_XER,r10;                                           \
        mtctr   r11;                                                    \
        stwcx.  r0,0,r1;                /* to clear the reservation */  \
        lwz     r11,_LINK(r1);                                          \
        mtlr    r11;                                                    \
        lwz     r10,_CCR(r1);                                           \
        mtcrf   0xff,r10;                                               \
        lwz     r9,_DEAR(r1);                                           \
        lwz     r10,_ESR(r1);                                           \
        mtspr   SPRN_DEAR,r9;                                           \
        mtspr   SPRN_ESR,r10;                                           \
        lwz     r11,_NIP(r1);                                           \
        lwz     r12,_MSR(r1);                                           \
        mtspr   exc_lvl_srr0,r11;                                       \
        mtspr   exc_lvl_srr1,r12;                                       \
        REST_GPRS(9, 12, r1);                                           \
        REST_GPR(1, r1);                                                \
        exc_lvl_rfi;                                                    \
        b       .;              /* prevent prefetch past exc_lvl_rfi */

#define RESTORE_xSRR(exc_lvl_srr0, exc_lvl_srr1)                        \
        lwz     r9,_##exc_lvl_srr0(r1);                                 \
        lwz     r10,_##exc_lvl_srr1(r1);                                \
        mtspr   SPRN_##exc_lvl_srr0,r9;                                 \
        mtspr   SPRN_##exc_lvl_srr1,r10;

#if defined(CONFIG_PPC_E500)
#ifdef CONFIG_PHYS_64BIT
#define RESTORE_MAS7                                                    \
        lwz     r11,MAS7(r1);                                           \
        mtspr   SPRN_MAS7,r11;
#else
#define RESTORE_MAS7
#endif /* CONFIG_PHYS_64BIT */
#define RESTORE_MMU_REGS                                                \
        lwz     r9,MAS0(r1);                                            \
        lwz     r10,MAS1(r1);                                           \
        lwz     r11,MAS2(r1);                                           \
        mtspr   SPRN_MAS0,r9;                                           \
        lwz     r9,MAS3(r1);                                            \
        mtspr   SPRN_MAS1,r10;                                          \
        lwz     r10,MAS6(r1);                                           \
        mtspr   SPRN_MAS2,r11;                                          \
        mtspr   SPRN_MAS3,r9;                                           \
        mtspr   SPRN_MAS6,r10;                                          \
        RESTORE_MAS7;
#elif defined(CONFIG_44x)
#define RESTORE_MMU_REGS                                                \
        lwz     r9,MMUCR(r1);                                           \
        mtspr   SPRN_MMUCR,r9;
#else
#define RESTORE_MMU_REGS
#endif

        .globl  ret_from_crit_exc
ret_from_crit_exc:
        RESTORE_xSRR(SRR0,SRR1);
        RESTORE_MMU_REGS;
        RET_FROM_EXC_LEVEL(SPRN_CSRR0, SPRN_CSRR1, PPC_RFCI)
_ASM_NOKPROBE_SYMBOL(ret_from_crit_exc)

        .globl  ret_from_debug_exc
ret_from_debug_exc:
        RESTORE_xSRR(SRR0,SRR1);
        RESTORE_xSRR(CSRR0,CSRR1);
        RESTORE_MMU_REGS;
        RET_FROM_EXC_LEVEL(SPRN_DSRR0, SPRN_DSRR1, PPC_RFDI)
_ASM_NOKPROBE_SYMBOL(ret_from_debug_exc)

        .globl  ret_from_mcheck_exc
ret_from_mcheck_exc:
        RESTORE_xSRR(SRR0,SRR1);
        RESTORE_xSRR(CSRR0,CSRR1);
        RESTORE_xSRR(DSRR0,DSRR1);
        RESTORE_MMU_REGS;
        RET_FROM_EXC_LEVEL(SPRN_MCSRR0, SPRN_MCSRR1, PPC_RFMCI)
_ASM_NOKPROBE_SYMBOL(ret_from_mcheck_exc)
#endif /* CONFIG_BOOKE */