#include <asm/asm-offsets.h>
#include <asm/bug.h>
#ifdef CONFIG_PPC_BOOK3S
#include <asm/exception-64s.h>
#else
#include <asm/exception-64e.h>
#endif
#include <asm/feature-fixups.h>
#include <asm/head-64.h>
#include <asm/hw_irq.h>
#include <asm/kup.h>
#include <asm/mmu.h>
#include <asm/ppc_asm.h>
#include <asm/ptrace.h>

        .align 7

.macro DEBUG_SRR_VALID srr
#ifdef CONFIG_PPC_RFI_SRR_DEBUG
        .ifc \srr,srr
        mfspr   r11,SPRN_SRR0
        ld      r12,_NIP(r1)
        clrrdi  r11,r11,2
        clrrdi  r12,r12,2
100:    tdne    r11,r12
        EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
        mfspr   r11,SPRN_SRR1
        ld      r12,_MSR(r1)
100:    tdne    r11,r12
        EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
        .else
        mfspr   r11,SPRN_HSRR0
        ld      r12,_NIP(r1)
        clrrdi  r11,r11,2
        clrrdi  r12,r12,2
100:    tdne    r11,r12
        EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
        mfspr   r11,SPRN_HSRR1
        ld      r12,_MSR(r1)
100:    tdne    r11,r12
        EMIT_WARN_ENTRY 100b,__FILE__,__LINE__,(BUGFLAG_WARNING | BUGFLAG_ONCE)
        .endif
#endif
.endm

#ifdef CONFIG_PPC_BOOK3S
.macro system_call_vectored name trapnr
        .globl system_call_vectored_\name
system_call_vectored_\name:
_ASM_NOKPROBE_SYMBOL(system_call_vectored_\name)
        SCV_INTERRUPT_TO_KERNEL
        mr      r10,r1
        ld      r1,PACAKSAVE(r13)
        std     r10,0(r1)
        std     r11,_LINK(r1)
        std     r11,_NIP(r1)    /* Saved LR is also the next instruction */
        std     r12,_MSR(r1)
        std     r0,GPR0(r1)
        std     r10,GPR1(r1)
        std     r2,GPR2(r1)
        LOAD_PACA_TOC()
        mfcr    r12
        li      r11,0
        /* Save syscall parameters in r3-r8 */
        SAVE_GPRS(3, 8, r1)
        /* Zero r9-r12, this should only be required when restoring all GPRs */
        std     r11,GPR9(r1)
        std     r11,GPR10(r1)
        std     r11,GPR11(r1)
        std     r11,GPR12(r1)
        std     r9,GPR13(r1)
        SAVE_NVGPRS(r1)
        std     r11,_XER(r1)
        std     r11,_CTR(r1)

        li      r11,\trapnr
        std     r11,_TRAP(r1)
        std     r12,_CCR(r1)
        std     r3,ORIG_GPR3(r1)
        LOAD_REG_IMMEDIATE(r11, STACK_FRAME_REGS_MARKER)
        std     r11,STACK_INT_FRAME_MARKER(r1)          /* "regs" marker */
        /* Calling convention has r3 = regs, r4 = orig r0 */
        addi    r3,r1,STACK_INT_FRAME_REGS
        mr      r4,r0

BEGIN_FTR_SECTION
        HMT_MEDIUM
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)

        /*
         * scv enters with MSR[EE]=1 and is immediately considered soft-masked.
         * The entry vector already sets PACAIRQSOFTMASK to IRQS_ALL_DISABLED,
         * and interrupts may be masked and pending already.
         * system_call_exception() will call trace_hardirqs_off() which means
         * interrupts could already have been blocked before trace_hardirqs_off,
         * but this is the best we can do.
         */

        /*
         * Zero user registers to prevent influencing speculative execution
         * state of kernel code.
         */
        SANITIZE_SYSCALL_GPRS()
        bl      CFUNC(system_call_exception)

.Lsyscall_vectored_\name\()_exit:
        addi    r4,r1,STACK_INT_FRAME_REGS
        li      r5,1 /* scv */
        bl      CFUNC(syscall_exit_prepare)
        std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
.Lsyscall_vectored_\name\()_rst_start:
        lbz     r11,PACAIRQHAPPENED(r13)
        andi.   r11,r11,(~PACA_IRQ_HARD_DIS)@l
        bne-    syscall_vectored_\name\()_restart
        li      r11,IRQS_ENABLED
        stb     r11,PACAIRQSOFTMASK(r13)
        li      r11,0
        stb     r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS

        ld      r2,_CCR(r1)
        ld      r4,_NIP(r1)
        ld      r5,_MSR(r1)

BEGIN_FTR_SECTION
        stdcx.  r0,0,r1                 /* to clear the reservation */
END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)

BEGIN_FTR_SECTION
        HMT_MEDIUM_LOW
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)

        SANITIZE_RESTORE_NVGPRS()
        cmpdi   r3,0
        bne     .Lsyscall_vectored_\name\()_restore_regs

        /* rfscv returns with LR->NIA and CTR->MSR */
        mtlr    r4
        mtctr   r5

        /* Could zero these as per ABI, but we may consider a stricter ABI
         * which preserves these if libc implementations can benefit, so
         * restore them for now until further measurement is done. */
        REST_GPR(0, r1)
        REST_GPRS(4, 8, r1)
        /* Zero volatile regs that may contain sensitive kernel data */
        ZEROIZE_GPRS(9, 12)
        mtspr   SPRN_XER,r0

        /*
         * We don't need to restore AMR on the way back to userspace for KUAP.
         * The value of AMR only matters while we're in the kernel.
         */
        mtcr    r2
        REST_GPRS(2, 3, r1)
        REST_GPR(13, r1)
        REST_GPR(1, r1)
        RFSCV_TO_USER
        b       .       /* prevent speculative execution */

.Lsyscall_vectored_\name\()_restore_regs:
        mtspr   SPRN_SRR0,r4
        mtspr   SPRN_SRR1,r5

        ld      r3,_CTR(r1)
        ld      r4,_LINK(r1)
        ld      r5,_XER(r1)

        HANDLER_RESTORE_NVGPRS()
        REST_GPR(0, r1)
        mtcr    r2
        mtctr   r3
        mtlr    r4
        mtspr   SPRN_XER,r5
        REST_GPRS(2, 13, r1)
        REST_GPR(1, r1)
        RFI_TO_USER
.Lsyscall_vectored_\name\()_rst_end:

syscall_vectored_\name\()_restart:
_ASM_NOKPROBE_SYMBOL(syscall_vectored_\name\()_restart)
        GET_PACA(r13)
        ld      r1,PACA_EXIT_SAVE_R1(r13)
        LOAD_PACA_TOC()
        ld      r3,RESULT(r1)
        addi    r4,r1,STACK_INT_FRAME_REGS
        li      r11,IRQS_ALL_DISABLED
        stb     r11,PACAIRQSOFTMASK(r13)
        bl      CFUNC(syscall_exit_restart)
        std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
        b       .Lsyscall_vectored_\name\()_rst_start
1:

SOFT_MASK_TABLE(.Lsyscall_vectored_\name\()_rst_start, 1b)
RESTART_TABLE(.Lsyscall_vectored_\name\()_rst_start, .Lsyscall_vectored_\name\()_rst_end, syscall_vectored_\name\()_restart)

.endm

system_call_vectored common 0x3000

/*
 * We instantiate another entry copy for the SIGILL variant, with TRAP=0x7ff0
 * which is tested by system_call_exception when r0 is -1 (as set by vector
 * entry code).
 */
system_call_vectored sigill 0x7ff0

#endif /* CONFIG_PPC_BOOK3S */

        .balign IFETCH_ALIGN_BYTES
        .globl system_call_common_real
system_call_common_real:
_ASM_NOKPROBE_SYMBOL(system_call_common_real)
        ld      r10,PACAKMSR(r13)       /* get MSR value for kernel */
        mtmsrd  r10

        .balign IFETCH_ALIGN_BYTES
        .globl system_call_common
system_call_common:
_ASM_NOKPROBE_SYMBOL(system_call_common)
        mr      r10,r1
        ld      r1,PACAKSAVE(r13)
        std     r10,0(r1)
        std     r11,_NIP(r1)
        std     r12,_MSR(r1)
        std     r0,GPR0(r1)
        std     r10,GPR1(r1)
        std     r2,GPR2(r1)
#ifdef CONFIG_PPC_E500
START_BTB_FLUSH_SECTION
        BTB_FLUSH(r10)
END_BTB_FLUSH_SECTION
#endif
        LOAD_PACA_TOC()
        mfcr    r12
        li      r11,0
        /* Save syscall parameters in r3-r8 */
        SAVE_GPRS(3, 8, r1)
        /* Zero r9-r12, this should only be required when restoring all GPRs */
        std     r11,GPR9(r1)
        std     r11,GPR10(r1)
        std     r11,GPR11(r1)
        std     r11,GPR12(r1)
        std     r9,GPR13(r1)
        SAVE_NVGPRS(r1)
        std     r11,_XER(r1)
        std     r11,_CTR(r1)
        mflr    r10

        /*
         * This clears CR0.SO (bit 28), which is the error indication on
         * return from this system call.
         */
        rldimi  r12,r11,28,(63-28)
        li      r11,0xc00
        std     r10,_LINK(r1)
        std     r11,_TRAP(r1)
        std     r12,_CCR(r1)
        std     r3,ORIG_GPR3(r1)
        LOAD_REG_IMMEDIATE(r11, STACK_FRAME_REGS_MARKER)
        std     r11,STACK_INT_FRAME_MARKER(r1)          /* "regs" marker */
        /* Calling convention has r3 = regs, r4 = orig r0 */
        addi    r3,r1,STACK_INT_FRAME_REGS
        mr      r4,r0

#ifdef CONFIG_PPC_BOOK3S
        li      r11,1
        stb     r11,PACASRR_VALID(r13)
#endif

        /*
         * We always enter kernel from userspace with irq soft-mask enabled and
         * nothing pending. system_call_exception() will call
         * trace_hardirqs_off().
         */
        li      r11,IRQS_ALL_DISABLED
        stb     r11,PACAIRQSOFTMASK(r13)
#ifdef CONFIG_PPC_BOOK3S
        li      r12,-1 /* Set MSR_EE and MSR_RI */
        mtmsrd  r12,1
#else
        wrteei  1
#endif

        /*
         * Zero user registers to prevent influencing speculative execution
         * state of kernel code.
         */
        SANITIZE_SYSCALL_GPRS()
        bl      CFUNC(system_call_exception)

.Lsyscall_exit:
        addi    r4,r1,STACK_INT_FRAME_REGS
        li      r5,0 /* !scv */
        bl      CFUNC(syscall_exit_prepare)
        std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
#ifdef CONFIG_PPC_BOOK3S
.Lsyscall_rst_start:
        lbz     r11,PACAIRQHAPPENED(r13)
        andi.   r11,r11,(~PACA_IRQ_HARD_DIS)@l
        bne-    syscall_restart
#endif
        li      r11,IRQS_ENABLED
        stb     r11,PACAIRQSOFTMASK(r13)
        li      r11,0
        stb     r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS

        ld      r2,_CCR(r1)
        ld      r6,_LINK(r1)
        mtlr    r6

#ifdef CONFIG_PPC_BOOK3S
        lbz     r4,PACASRR_VALID(r13)
        cmpdi   r4,0
        bne     1f
        li      r4,0
        stb     r4,PACASRR_VALID(r13)
#endif
        ld      r4,_NIP(r1)
        ld      r5,_MSR(r1)
        mtspr   SPRN_SRR0,r4
        mtspr   SPRN_SRR1,r5
1:
        DEBUG_SRR_VALID srr

BEGIN_FTR_SECTION
        stdcx.  r0,0,r1                 /* to clear the reservation */
END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)

        SANITIZE_RESTORE_NVGPRS()
        cmpdi   r3,0
        bne     .Lsyscall_restore_regs
        /* Zero volatile regs that may contain sensitive kernel data */
        ZEROIZE_GPR(0)
        ZEROIZE_GPRS(4, 12)
        mtctr   r0
        mtspr   SPRN_XER,r0
.Lsyscall_restore_regs_cont:

BEGIN_FTR_SECTION
        HMT_MEDIUM_LOW
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)

        /*
         * We don't need to restore AMR on the way back to userspace for KUAP.
         * The value of AMR only matters while we're in the kernel.
         */
        mtcr    r2
        REST_GPRS(2, 3, r1)
        REST_GPR(13, r1)
        REST_GPR(1, r1)
        RFI_TO_USER
        b       .       /* prevent speculative execution */

.Lsyscall_restore_regs:
        ld      r3,_CTR(r1)
        ld      r4,_XER(r1)
        HANDLER_RESTORE_NVGPRS()
        mtctr   r3
        mtspr   SPRN_XER,r4
        REST_GPR(0, r1)
        REST_GPRS(4, 12, r1)
        b       .Lsyscall_restore_regs_cont
.Lsyscall_rst_end:

#ifdef CONFIG_PPC_BOOK3S
syscall_restart:
_ASM_NOKPROBE_SYMBOL(syscall_restart)
        GET_PACA(r13)
        ld      r1,PACA_EXIT_SAVE_R1(r13)
        LOAD_PACA_TOC()
        ld      r3,RESULT(r1)
        addi    r4,r1,STACK_INT_FRAME_REGS
        li      r11,IRQS_ALL_DISABLED
        stb     r11,PACAIRQSOFTMASK(r13)
        bl      CFUNC(syscall_exit_restart)
        std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
        b       .Lsyscall_rst_start
1:

SOFT_MASK_TABLE(.Lsyscall_rst_start, 1b)
RESTART_TABLE(.Lsyscall_rst_start, .Lsyscall_rst_end, syscall_restart)
#endif

        /*
         * If MSR EE/RI was never enabled, IRQs not reconciled, NVGPRs not
         * touched, no exit work created, then this can be used.
         */
        .balign IFETCH_ALIGN_BYTES
        .globl fast_interrupt_return_srr
fast_interrupt_return_srr:
_ASM_NOKPROBE_SYMBOL(fast_interrupt_return_srr)
        kuap_check_amr r3, r4
        ld      r5,_MSR(r1)
        andi.   r0,r5,MSR_PR
#ifdef CONFIG_PPC_BOOK3S
        beq     1f
        kuap_user_restore r3, r4
        b       .Lfast_user_interrupt_return_srr
1:      kuap_kernel_restore r3, r4
        andi.   r0,r5,MSR_RI
        li      r3,0 /* 0 return value, no EMULATE_STACK_STORE */
        bne+    .Lfast_kernel_interrupt_return_srr
        addi    r3,r1,STACK_INT_FRAME_REGS
        bl      CFUNC(unrecoverable_exception)
        b       . /* should not get here */
#else
        bne     .Lfast_user_interrupt_return_srr
        b       .Lfast_kernel_interrupt_return_srr
#endif

.macro interrupt_return_macro srr
        .balign IFETCH_ALIGN_BYTES
        .globl interrupt_return_\srr
interrupt_return_\srr\():
_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\())
        ld      r4,_MSR(r1)
        andi.   r0,r4,MSR_PR
        beq     interrupt_return_\srr\()_kernel
interrupt_return_\srr\()_user: /* make backtraces match the _kernel variant */
_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user)
        addi    r3,r1,STACK_INT_FRAME_REGS
        bl      CFUNC(interrupt_exit_user_prepare)
#ifndef CONFIG_INTERRUPT_SANITIZE_REGISTERS
        cmpdi   r3,0
        bne-    .Lrestore_nvgprs_\srr
.Lrestore_nvgprs_\srr\()_cont:
#endif
        std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
#ifdef CONFIG_PPC_BOOK3S
.Linterrupt_return_\srr\()_user_rst_start:
        lbz     r11,PACAIRQHAPPENED(r13)
        andi.   r11,r11,(~PACA_IRQ_HARD_DIS)@l
        bne-    interrupt_return_\srr\()_user_restart
#endif
        li      r11,IRQS_ENABLED
        stb     r11,PACAIRQSOFTMASK(r13)
        li      r11,0
        stb     r11,PACAIRQHAPPENED(r13) # clear out possible HARD_DIS

.Lfast_user_interrupt_return_\srr\():
        SANITIZE_RESTORE_NVGPRS()
#ifdef CONFIG_PPC_BOOK3S
        .ifc \srr,srr
        lbz     r4,PACASRR_VALID(r13)
        .else
        lbz     r4,PACAHSRR_VALID(r13)
        .endif
        cmpdi   r4,0
        li      r4,0
        bne     1f
#endif
        ld      r11,_NIP(r1)
        ld      r12,_MSR(r1)
        .ifc \srr,srr
        mtspr   SPRN_SRR0,r11
        mtspr   SPRN_SRR1,r12
1:
#ifdef CONFIG_PPC_BOOK3S
        stb     r4,PACASRR_VALID(r13)
#endif
        .else
        mtspr   SPRN_HSRR0,r11
        mtspr   SPRN_HSRR1,r12
1:
#ifdef CONFIG_PPC_BOOK3S
        stb     r4,PACAHSRR_VALID(r13)
#endif
        .endif
        DEBUG_SRR_VALID \srr

#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG
        lbz     r4,PACAIRQSOFTMASK(r13)
        tdnei   r4,IRQS_ENABLED
#endif

BEGIN_FTR_SECTION
        ld      r10,_PPR(r1)
        mtspr   SPRN_PPR,r10
END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)

BEGIN_FTR_SECTION
        stdcx.  r0,0,r1         /* to clear the reservation */
FTR_SECTION_ELSE
        ldarx   r0,0,r1
ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)

        ld      r3,_CCR(r1)
        ld      r4,_LINK(r1)
        ld      r5,_CTR(r1)
        ld      r6,_XER(r1)
        li      r0,0

        REST_GPRS(7, 13, r1)

        mtcr    r3
        mtlr    r4
        mtctr   r5
        mtspr   SPRN_XER,r6

        REST_GPRS(2, 6, r1)
        REST_GPR(0, r1)
        REST_GPR(1, r1)
        .ifc \srr,srr
        RFI_TO_USER
        .else
        HRFI_TO_USER
        .endif
        b       .       /* prevent speculative execution */
.Linterrupt_return_\srr\()_user_rst_end:

#ifndef CONFIG_INTERRUPT_SANITIZE_REGISTERS
.Lrestore_nvgprs_\srr\():
        REST_NVGPRS(r1)
        b       .Lrestore_nvgprs_\srr\()_cont
#endif

#ifdef CONFIG_PPC_BOOK3S
interrupt_return_\srr\()_user_restart:
_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_user_restart)
        GET_PACA(r13)
        ld      r1,PACA_EXIT_SAVE_R1(r13)
        LOAD_PACA_TOC()
        addi    r3,r1,STACK_INT_FRAME_REGS
        li      r11,IRQS_ALL_DISABLED
        stb     r11,PACAIRQSOFTMASK(r13)
        bl      CFUNC(interrupt_exit_user_restart)
        std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
        b       .Linterrupt_return_\srr\()_user_rst_start
1:

SOFT_MASK_TABLE(.Linterrupt_return_\srr\()_user_rst_start, 1b)
RESTART_TABLE(.Linterrupt_return_\srr\()_user_rst_start, .Linterrupt_return_\srr\()_user_rst_end, interrupt_return_\srr\()_user_restart)
#endif

        .balign IFETCH_ALIGN_BYTES
interrupt_return_\srr\()_kernel:
_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel)
        addi    r3,r1,STACK_INT_FRAME_REGS
        bl      CFUNC(interrupt_exit_kernel_prepare)

        std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
.Linterrupt_return_\srr\()_kernel_rst_start:
        ld      r11,SOFTE(r1)
        cmpwi   r11,IRQS_ENABLED
        stb     r11,PACAIRQSOFTMASK(r13)
        beq     .Linterrupt_return_\srr\()_soft_enabled

        /*
         * Returning to soft-disabled context.
         * Check if a MUST_HARD_MASK interrupt has become pending, in which
         * case we need to disable MSR[EE] in the return context.
         *
         * The MSR[EE] check catches among other things the short incoherency
         * in hard_irq_disable() between clearing MSR[EE] and setting
         * PACA_IRQ_HARD_DIS.
         */
        ld      r12,_MSR(r1)
        andi.   r10,r12,MSR_EE
        beq     .Lfast_kernel_interrupt_return_\srr\() // EE already disabled
        lbz     r11,PACAIRQHAPPENED(r13)
        andi.   r10,r11,PACA_IRQ_MUST_HARD_MASK
        bne     1f // HARD_MASK is pending
        // No HARD_MASK pending, clear possible HARD_DIS set by interrupt
        andi.   r11,r11,(~PACA_IRQ_HARD_DIS)@l
        stb     r11,PACAIRQHAPPENED(r13)
        b       .Lfast_kernel_interrupt_return_\srr\()


1:      /* Must clear MSR_EE from _MSR */
#ifdef CONFIG_PPC_BOOK3S
        li      r10,0
        /* Clear valid before changing _MSR */
        .ifc \srr,srr
        stb     r10,PACASRR_VALID(r13)
        .else
        stb     r10,PACAHSRR_VALID(r13)
        .endif
#endif
        xori    r12,r12,MSR_EE
        std     r12,_MSR(r1)
        b       .Lfast_kernel_interrupt_return_\srr\()

.Linterrupt_return_\srr\()_soft_enabled:
        /*
         * In the soft-enabled case, need to double-check that we have no
         * pending interrupts that might have come in before we reached the
         * restart section of code, and restart the exit so those can be
         * handled.
         *
         * If there are none, it is be possible that the interrupt still
         * has PACA_IRQ_HARD_DIS set, which needs to be cleared for the
         * interrupted context. This clear will not clobber a new pending
         * interrupt coming in, because we're in the restart section, so
         * such would return to the restart location.
         */
#ifdef CONFIG_PPC_BOOK3S
        lbz     r11,PACAIRQHAPPENED(r13)
        andi.   r11,r11,(~PACA_IRQ_HARD_DIS)@l
        bne-    interrupt_return_\srr\()_kernel_restart
#endif
        li      r11,0
        stb     r11,PACAIRQHAPPENED(r13) // clear the possible HARD_DIS

.Lfast_kernel_interrupt_return_\srr\():
        SANITIZE_RESTORE_NVGPRS()
        cmpdi   cr1,r3,0
#ifdef CONFIG_PPC_BOOK3S
        .ifc \srr,srr
        lbz     r4,PACASRR_VALID(r13)
        .else
        lbz     r4,PACAHSRR_VALID(r13)
        .endif
        cmpdi   r4,0
        li      r4,0
        bne     1f
#endif
        ld      r11,_NIP(r1)
        ld      r12,_MSR(r1)
        .ifc \srr,srr
        mtspr   SPRN_SRR0,r11
        mtspr   SPRN_SRR1,r12
1:
#ifdef CONFIG_PPC_BOOK3S
        stb     r4,PACASRR_VALID(r13)
#endif
        .else
        mtspr   SPRN_HSRR0,r11
        mtspr   SPRN_HSRR1,r12
1:
#ifdef CONFIG_PPC_BOOK3S
        stb     r4,PACAHSRR_VALID(r13)
#endif
        .endif
        DEBUG_SRR_VALID \srr

BEGIN_FTR_SECTION
        stdcx.  r0,0,r1         /* to clear the reservation */
FTR_SECTION_ELSE
        ldarx   r0,0,r1
ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)

        ld      r3,_LINK(r1)
        ld      r4,_CTR(r1)
        ld      r5,_XER(r1)
        ld      r6,_CCR(r1)
        li      r0,0

        REST_GPRS(7, 12, r1)

        mtlr    r3
        mtctr   r4
        mtspr   SPRN_XER,r5

        /*
         * Leaving a stale STACK_FRAME_REGS_MARKER on the stack can confuse
         * the reliable stack unwinder later on. Clear it.
         */
        std     r0,STACK_INT_FRAME_MARKER(r1)

        REST_GPRS(2, 5, r1)

        bne-    cr1,1f /* emulate stack store */
        mtcr    r6
        REST_GPR(6, r1)
        REST_GPR(0, r1)
        REST_GPR(1, r1)
        .ifc \srr,srr
        RFI_TO_KERNEL
        .else
        HRFI_TO_KERNEL
        .endif
        b       .       /* prevent speculative execution */

1:      /*
         * Emulate stack store with update. New r1 value was already calculated
         * and updated in our interrupt regs by emulate_loadstore, but we can't
         * store the previous value of r1 to the stack before re-loading our
         * registers from it, otherwise they could be clobbered.  Use
         * PACA_EXGEN as temporary storage to hold the store data, as
         * interrupts are disabled here so it won't be clobbered.
         */
        mtcr    r6
        std     r9,PACA_EXGEN+0(r13)
        addi    r9,r1,INT_FRAME_SIZE /* get original r1 */
        REST_GPR(6, r1)
        REST_GPR(0, r1)
        REST_GPR(1, r1)
        std     r9,0(r1) /* perform store component of stdu */
        ld      r9,PACA_EXGEN+0(r13)

        .ifc \srr,srr
        RFI_TO_KERNEL
        .else
        HRFI_TO_KERNEL
        .endif
        b       .       /* prevent speculative execution */
.Linterrupt_return_\srr\()_kernel_rst_end:

#ifdef CONFIG_PPC_BOOK3S
interrupt_return_\srr\()_kernel_restart:
_ASM_NOKPROBE_SYMBOL(interrupt_return_\srr\()_kernel_restart)
        GET_PACA(r13)
        ld      r1,PACA_EXIT_SAVE_R1(r13)
        LOAD_PACA_TOC()
        addi    r3,r1,STACK_INT_FRAME_REGS
        li      r11,IRQS_ALL_DISABLED
        stb     r11,PACAIRQSOFTMASK(r13)
        bl      CFUNC(interrupt_exit_kernel_restart)
        std     r1,PACA_EXIT_SAVE_R1(r13) /* save r1 for restart */
        b       .Linterrupt_return_\srr\()_kernel_rst_start
1:

SOFT_MASK_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, 1b)
RESTART_TABLE(.Linterrupt_return_\srr\()_kernel_rst_start, .Linterrupt_return_\srr\()_kernel_rst_end, interrupt_return_\srr\()_kernel_restart)
#endif

.endm

interrupt_return_macro srr
#ifdef CONFIG_PPC_BOOK3S
interrupt_return_macro hsrr

        .globl __end_soft_masked
__end_soft_masked:
DEFINE_FIXED_SYMBOL(__end_soft_masked, text)
#endif /* CONFIG_PPC_BOOK3S */

#ifdef CONFIG_PPC_BOOK3S
_GLOBAL(ret_from_fork_scv)
        bl      CFUNC(schedule_tail)
        HANDLER_RESTORE_NVGPRS()
        li      r3,0    /* fork() return value */
        b       .Lsyscall_vectored_common_exit
#endif

_GLOBAL(ret_from_fork)
        bl      CFUNC(schedule_tail)
        HANDLER_RESTORE_NVGPRS()
        li      r3,0    /* fork() return value */
        b       .Lsyscall_exit

_GLOBAL(ret_from_kernel_user_thread)
        bl      CFUNC(schedule_tail)
        mtctr   r14
        mr      r3,r15
#ifdef CONFIG_PPC64_ELF_ABI_V2
        mr      r12,r14
#endif
        bctrl
        li      r3,0
        /*
         * It does not matter whether this returns via the scv or sc path
         * because it returns as execve() and therefore has no calling ABI
         * (i.e., it sets registers according to the exec()ed entry point).
         */
        b       .Lsyscall_exit

_GLOBAL(start_kernel_thread)
        bl      CFUNC(schedule_tail)
        mtctr   r14
        mr      r3,r15
#ifdef CONFIG_PPC64_ELF_ABI_V2
        mr      r12,r14
#endif
        bctrl
        /*
         * This must not return. We actually want to BUG here, not WARN,
         * because BUG will exit the process which is what the kernel thread
         * should have done, which may give some hope of continuing.
         */
100:    trap
        EMIT_BUG_ENTRY 100b,__FILE__,__LINE__,0