/* SPDX-License-Identifier: GPL-2.0 */
/* arch/sparc64/kernel/ktlb.S: Kernel mapping TLB miss handling.
 *
 * Copyright (C) 1995, 1997, 2005, 2008 David S. Miller <davem@davemloft.net>
 * Copyright (C) 1996 Eddie C. Dost        (ecd@brainaid.de)
 * Copyright (C) 1996 Miguel de Icaza      (miguel@nuclecu.unam.mx)
 * Copyright (C) 1996,98,99 Jakub Jelinek  (jj@sunsite.mff.cuni.cz)
 */

#include <linux/pgtable.h>
#include <asm/head.h>
#include <asm/asi.h>
#include <asm/page.h>
#include <asm/tsb.h>

        .text
        .align          32

kvmap_itlb:
        /* g6: TAG TARGET */
        mov             TLB_TAG_ACCESS, %g4
        ldxa            [%g4] ASI_IMMU, %g4

        /* The kernel executes in context zero, therefore we do not
         * need to clear the context ID bits out of %g4 here.
         */

        /* sun4v_itlb_miss branches here with the missing virtual
         * address already loaded into %g4
         */
kvmap_itlb_4v:

        /* Catch kernel NULL pointer calls.  */
        sethi           %hi(PAGE_SIZE), %g5
        cmp             %g4, %g5
        blu,pn          %xcc, kvmap_itlb_longpath
         nop

        KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_itlb_load)

kvmap_itlb_tsb_miss:
        sethi           %hi(LOW_OBP_ADDRESS), %g5
        cmp             %g4, %g5
        blu,pn          %xcc, kvmap_itlb_vmalloc_addr
         mov            0x1, %g5
        sllx            %g5, 32, %g5
        cmp             %g4, %g5
        blu,pn          %xcc, kvmap_itlb_obp
         nop

kvmap_itlb_vmalloc_addr:
        KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_itlb_longpath)

        TSB_LOCK_TAG(%g1, %g2, %g7)
        TSB_WRITE(%g1, %g5, %g6)

        /* fallthrough to TLB load */

kvmap_itlb_load:

661:    stxa            %g5, [%g0] ASI_ITLB_DATA_IN
        retry
        .section        .sun4v_2insn_patch, "ax"
        .word           661b
        nop
        nop
        .previous

        /* For sun4v the ASI_ITLB_DATA_IN store and the retry
         * instruction get nop'd out and we get here to branch
         * to the sun4v tlb load code.  The registers are setup
         * as follows:
         *
         * %g4: vaddr
         * %g5: PTE
         * %g6: TAG
         *
         * The sun4v TLB load wants the PTE in %g3 so we fix that
         * up here.
         */
        ba,pt           %xcc, sun4v_itlb_load
         mov            %g5, %g3

kvmap_itlb_longpath:

661:    rdpr    %pstate, %g5
        wrpr    %g5, PSTATE_AG | PSTATE_MG, %pstate
        .section .sun4v_2insn_patch, "ax"
        .word   661b
        SET_GL(1)
        nop
        .previous

        rdpr    %tpc, %g5
        ba,pt   %xcc, sparc64_realfault_common
         mov    FAULT_CODE_ITLB, %g4

kvmap_itlb_obp:
        OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_itlb_longpath)

        TSB_LOCK_TAG(%g1, %g2, %g7)

        TSB_WRITE(%g1, %g5, %g6)

        ba,pt           %xcc, kvmap_itlb_load
         nop

kvmap_dtlb_obp:
        OBP_TRANS_LOOKUP(%g4, %g5, %g2, %g3, kvmap_dtlb_longpath)

        TSB_LOCK_TAG(%g1, %g2, %g7)

        TSB_WRITE(%g1, %g5, %g6)

        ba,pt           %xcc, kvmap_dtlb_load
         nop

kvmap_linear_early:
        sethi           %hi(kern_linear_pte_xor), %g7
        ldx             [%g7 + %lo(kern_linear_pte_xor)], %g2
        ba,pt           %xcc, kvmap_dtlb_tsb4m_load
         xor            %g2, %g4, %g5

        .align          32
kvmap_dtlb_tsb4m_load:
        TSB_LOCK_TAG(%g1, %g2, %g7)
        TSB_WRITE(%g1, %g5, %g6)
        ba,pt           %xcc, kvmap_dtlb_load
         nop

kvmap_dtlb:
        /* %g6: TAG TARGET */
        mov             TLB_TAG_ACCESS, %g4
        ldxa            [%g4] ASI_DMMU, %g4

        /* The kernel executes in context zero, therefore we do not
         * need to clear the context ID bits out of %g4 here.
         */

        /* sun4v_dtlb_miss branches here with the missing virtual
         * address already loaded into %g4
         */
kvmap_dtlb_4v:
        brgez,pn        %g4, kvmap_dtlb_nonlinear
         nop

#ifdef CONFIG_DEBUG_PAGEALLOC
        /* Index through the base page size TSB even for linear
         * mappings when using page allocation debugging.
         */
        KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
#else
        /* Correct TAG_TARGET is already in %g6, check 4mb TSB.  */
        KERN_TSB4M_LOOKUP_TL1(%g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)
#endif
        /* Linear mapping TSB lookup failed.  Fallthrough to kernel
         * page table based lookup.
         */
        .globl          kvmap_linear_patch
kvmap_linear_patch:
        ba,a,pt         %xcc, kvmap_linear_early

kvmap_dtlb_vmalloc_addr:
        KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)

        TSB_LOCK_TAG(%g1, %g2, %g7)
        TSB_WRITE(%g1, %g5, %g6)

        /* fallthrough to TLB load */

kvmap_dtlb_load:

661:    stxa            %g5, [%g0] ASI_DTLB_DATA_IN     ! Reload TLB
        retry
        .section        .sun4v_2insn_patch, "ax"
        .word           661b
        nop
        nop
        .previous

        /* For sun4v the ASI_DTLB_DATA_IN store and the retry
         * instruction get nop'd out and we get here to branch
         * to the sun4v tlb load code.  The registers are setup
         * as follows:
         *
         * %g4: vaddr
         * %g5: PTE
         * %g6: TAG
         *
         * The sun4v TLB load wants the PTE in %g3 so we fix that
         * up here.
         */
        ba,pt           %xcc, sun4v_dtlb_load
         mov            %g5, %g3

#ifdef CONFIG_SPARSEMEM_VMEMMAP
kvmap_vmemmap:
        KERN_PGTABLE_WALK(%g4, %g5, %g2, kvmap_dtlb_longpath)
        ba,a,pt         %xcc, kvmap_dtlb_load
#endif

kvmap_dtlb_nonlinear:
        /* Catch kernel NULL pointer derefs.  */
        sethi           %hi(PAGE_SIZE), %g5
        cmp             %g4, %g5
        bleu,pn         %xcc, kvmap_dtlb_longpath
         nop

#ifdef CONFIG_SPARSEMEM_VMEMMAP
        /* Do not use the TSB for vmemmap.  */
        sethi           %hi(VMEMMAP_BASE), %g5
        ldx             [%g5 + %lo(VMEMMAP_BASE)], %g5
        cmp             %g4,%g5
        bgeu,pn         %xcc, kvmap_vmemmap
         nop
#endif

        KERN_TSB_LOOKUP_TL1(%g4, %g6, %g5, %g1, %g2, %g3, kvmap_dtlb_load)

kvmap_dtlb_tsbmiss:
        sethi           %hi(MODULES_VADDR), %g5
        cmp             %g4, %g5
        blu,pn          %xcc, kvmap_dtlb_longpath
         sethi          %hi(VMALLOC_END), %g5
        ldx             [%g5 + %lo(VMALLOC_END)], %g5
        cmp             %g4, %g5
        bgeu,pn         %xcc, kvmap_dtlb_longpath
         nop

kvmap_check_obp:
        sethi           %hi(LOW_OBP_ADDRESS), %g5
        cmp             %g4, %g5
        blu,pn          %xcc, kvmap_dtlb_vmalloc_addr
         mov            0x1, %g5
        sllx            %g5, 32, %g5
        cmp             %g4, %g5
        blu,pn          %xcc, kvmap_dtlb_obp
         nop
        ba,pt           %xcc, kvmap_dtlb_vmalloc_addr
         nop

kvmap_dtlb_longpath:

661:    rdpr    %pstate, %g5
        wrpr    %g5, PSTATE_AG | PSTATE_MG, %pstate
        .section .sun4v_2insn_patch, "ax"
        .word   661b
        SET_GL(1)
        ldxa            [%g0] ASI_SCRATCHPAD, %g5
        .previous

        rdpr    %tl, %g3
        cmp     %g3, 1

661:    mov     TLB_TAG_ACCESS, %g4
        ldxa    [%g4] ASI_DMMU, %g5
        .section .sun4v_2insn_patch, "ax"
        .word   661b
        ldx     [%g5 + HV_FAULT_D_ADDR_OFFSET], %g5
        nop
        .previous

        /* The kernel executes in context zero, therefore we do not
         * need to clear the context ID bits out of %g5 here.
         */

        be,pt   %xcc, sparc64_realfault_common
         mov    FAULT_CODE_DTLB, %g4
        ba,pt   %xcc, winfix_trampoline
         nop