/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
 */
#include <asm/asm.h>
#include <asm/loongarch.h>
#include <asm/page.h>
#include <asm/pgtable.h>
#include <asm/regdef.h>
#include <asm/stackframe.h>

#define INVTLB_ADDR_GFALSE_AND_ASID     5

#define PTRS_PER_PGD_BITS       (PAGE_SHIFT - PTRLOG)
#define PTRS_PER_PUD_BITS       (PAGE_SHIFT - PTRLOG)
#define PTRS_PER_PMD_BITS       (PAGE_SHIFT - PTRLOG)
#define PTRS_PER_PTE_BITS       (PAGE_SHIFT - PTRLOG)

#ifdef CONFIG_32BIT
#define PTE_LL  ll.w
#define PTE_SC  sc.w
#else
#define PTE_LL  ll.d
#define PTE_SC  sc.d
#endif

        .macro tlb_do_page_fault, write
        SYM_CODE_START(tlb_do_page_fault_\write)
        UNWIND_HINT_UNDEFINED
        SAVE_ALL
        csrrd           a2, LOONGARCH_CSR_BADV
        move            a0, sp
        REG_S           a2, sp, PT_BVADDR
        li.w            a1, \write
        bl              do_page_fault
        RESTORE_ALL_AND_RET
        SYM_CODE_END(tlb_do_page_fault_\write)
        .endm

        tlb_do_page_fault 0
        tlb_do_page_fault 1

SYM_CODE_START(handle_tlb_protect)
        UNWIND_HINT_UNDEFINED
        BACKUP_T0T1
        SAVE_ALL
        move            a0, sp
        move            a1, zero
        csrrd           a2, LOONGARCH_CSR_BADV
        REG_S           a2, sp, PT_BVADDR
        la_abs          t0, do_page_fault
        jirl            ra, t0, 0
        RESTORE_ALL_AND_RET
SYM_CODE_END(handle_tlb_protect)

SYM_CODE_START(handle_tlb_load)
        UNWIND_HINT_UNDEFINED
        csrwr           t0, EXCEPTION_KS0
        csrwr           t1, EXCEPTION_KS1
        csrwr           ra, EXCEPTION_KS2

        /*
         * The vmalloc handling is not in the hotpath.
         */
        csrrd           t0, LOONGARCH_CSR_BADV
        bltz            t0, vmalloc_load
        csrrd           t1, LOONGARCH_CSR_PGDL

vmalloc_done_load:
        /* Get PGD offset in bytes */
#ifdef CONFIG_32BIT
        PTR_BSTRPICK    ra, t0, 31, PGDIR_SHIFT
#else
        PTR_BSTRPICK    ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT
#endif
        PTR_ALSL        t1, ra, t1, _PGD_T_LOG2

#if CONFIG_PGTABLE_LEVELS > 3
        PTR_L           t1, t1, 0
        PTR_BSTRPICK    ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT
        PTR_ALSL        t1, ra, t1, _PMD_T_LOG2

#endif
#if CONFIG_PGTABLE_LEVELS > 2
        PTR_L           t1, t1, 0
        PTR_BSTRPICK    ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT
        PTR_ALSL        t1, ra, t1, _PMD_T_LOG2

#endif
        PTR_L           ra, t1, 0

        /*
         * For huge tlb entries, pmde doesn't contain an address but
         * instead contains the tlb pte. Check the PAGE_HUGE bit and
         * see if we need to jump to huge tlb processing.
         */
        PTR_ROTRI       ra, ra, _PAGE_HUGE_SHIFT + 1
        bltz            ra, tlb_huge_update_load

        PTR_ROTRI       ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1)
        PTR_BSTRPICK    t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT
        PTR_ALSL        t1, t0, ra, _PTE_T_LOG2

#ifdef CONFIG_SMP
smp_pgtable_change_load:
        PTE_LL          t0, t1, 0
#else
        PTR_L           t0, t1, 0
#endif
        andi            ra, t0, _PAGE_PRESENT
        beqz            ra, nopage_tlb_load

        ori             t0, t0, _PAGE_VALID

#ifdef CONFIG_SMP
        PTE_SC          t0, t1, 0
        beqz            t0, smp_pgtable_change_load
#else
        PTR_S           t0, t1, 0
#endif

        tlbsrch
        PTR_BSTRINS     t1, zero, _PTE_T_LOG2, _PTE_T_LOG2
        PTR_L           t0, t1, 0
        PTR_L           t1, t1, _PTE_T_SIZE
        csrwr           t0, LOONGARCH_CSR_TLBELO0
        csrwr           t1, LOONGARCH_CSR_TLBELO1
        tlbwr

        csrrd           t0, EXCEPTION_KS0
        csrrd           t1, EXCEPTION_KS1
        csrrd           ra, EXCEPTION_KS2
        ertn

vmalloc_load:
        la_abs          t1, swapper_pg_dir
        b               vmalloc_done_load

        /* This is the entry point of a huge page. */
tlb_huge_update_load:
#ifdef CONFIG_SMP
        PTE_LL          ra, t1, 0
#else
        PTR_ROTRI       ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1)
#endif
        andi            t0, ra, _PAGE_PRESENT
        beqz            t0, nopage_tlb_load

#ifdef CONFIG_SMP
        ori             t0, ra, _PAGE_VALID
        PTE_SC          t0, t1, 0
        beqz            t0, tlb_huge_update_load
        ori             t0, ra, _PAGE_VALID
#else
        ori             t0, ra, _PAGE_VALID
        PTR_S           t0, t1, 0
#endif
        csrrd           ra, LOONGARCH_CSR_ASID
        csrrd           t1, LOONGARCH_CSR_BADV
        andi            ra, ra, CSR_ASID_ASID
        invtlb          INVTLB_ADDR_GFALSE_AND_ASID, ra, t1

        /*
         * A huge PTE describes an area the size of the
         * configured huge page size. This is twice the
         * of the large TLB entry size we intend to use.
         * A TLB entry half the size of the configured
         * huge page size is configured into entrylo0
         * and entrylo1 to cover the contiguous huge PTE
         * address space.
         */
        /* Huge page: Move Global bit */
        xori            t0, t0, _PAGE_HUGE
        lu12i.w         t1, _PAGE_HGLOBAL >> 12
        and             t1, t0, t1
        PTR_SRLI        t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
        or              t0, t0, t1

        move            ra, t0
        csrwr           ra, LOONGARCH_CSR_TLBELO0

        /* Convert to entrylo1 */
        PTR_ADDI        t1, zero, 1
        PTR_SLLI        t1, t1, (HPAGE_SHIFT - 1)
        PTR_ADD         t0, t0, t1
        csrwr           t0, LOONGARCH_CSR_TLBELO1

        /* Set huge page tlb entry size */
        PTR_LI          t0, (CSR_TLBIDX_PS >> 16) << 16
        PTR_LI          t1, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT))
        csrxchg         t1, t0, LOONGARCH_CSR_TLBIDX

        tlbfill

        PTR_LI          t0, (CSR_TLBIDX_PS >> 16) << 16
        PTR_LI          t1, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT))
        csrxchg         t1, t0, LOONGARCH_CSR_TLBIDX

        csrrd           t0, EXCEPTION_KS0
        csrrd           t1, EXCEPTION_KS1
        csrrd           ra, EXCEPTION_KS2
        ertn

nopage_tlb_load:
        dbar            0x700
        csrrd           ra, EXCEPTION_KS2
        la_abs          t0, tlb_do_page_fault_0
        jr              t0
SYM_CODE_END(handle_tlb_load)

SYM_CODE_START(handle_tlb_load_ptw)
        UNWIND_HINT_UNDEFINED
        csrwr           t0, LOONGARCH_CSR_KS0
        csrwr           t1, LOONGARCH_CSR_KS1
        la_abs          t0, tlb_do_page_fault_0
        jr              t0
SYM_CODE_END(handle_tlb_load_ptw)

SYM_CODE_START(handle_tlb_store)
        UNWIND_HINT_UNDEFINED
        csrwr           t0, EXCEPTION_KS0
        csrwr           t1, EXCEPTION_KS1
        csrwr           ra, EXCEPTION_KS2

        /*
         * The vmalloc handling is not in the hotpath.
         */
        csrrd           t0, LOONGARCH_CSR_BADV
        bltz            t0, vmalloc_store
        csrrd           t1, LOONGARCH_CSR_PGDL

vmalloc_done_store:
        /* Get PGD offset in bytes */
#ifdef CONFIG_32BIT
        PTR_BSTRPICK    ra, t0, 31, PGDIR_SHIFT
#else
        PTR_BSTRPICK    ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT
#endif
        PTR_ALSL        t1, ra, t1, _PGD_T_LOG2

#if CONFIG_PGTABLE_LEVELS > 3
        PTR_L           t1, t1, 0
        PTR_BSTRPICK    ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT
        PTR_ALSL        t1, ra, t1, _PMD_T_LOG2
#endif
#if CONFIG_PGTABLE_LEVELS > 2
        PTR_L           t1, t1, 0
        PTR_BSTRPICK    ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT
        PTR_ALSL        t1, ra, t1, _PMD_T_LOG2
#endif
        PTR_L           ra, t1, 0

        /*
         * For huge tlb entries, pmde doesn't contain an address but
         * instead contains the tlb pte. Check the PAGE_HUGE bit and
         * see if we need to jump to huge tlb processing.
         */
        PTR_ROTRI       ra, ra, _PAGE_HUGE_SHIFT + 1
        bltz            ra, tlb_huge_update_store

        PTR_ROTRI       ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1)
        PTR_BSTRPICK    t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT
        PTR_ALSL        t1, t0, ra, _PTE_T_LOG2

#ifdef CONFIG_SMP
smp_pgtable_change_store:
        PTE_LL          t0, t1, 0
#else
        PTR_L           t0, t1, 0
#endif

#ifdef CONFIG_64BIT
        andi            ra, t0, _PAGE_PRESENT | _PAGE_WRITE
        xori            ra, ra, _PAGE_PRESENT | _PAGE_WRITE
#else
        PTR_LI          ra, _PAGE_PRESENT | _PAGE_WRITE
        and             ra, ra, t0
        nor             ra, ra, zero
#endif
        bnez            ra, nopage_tlb_store

#ifdef CONFIG_64BIT
        ori             t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
#else
        PTR_LI          ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
        or              t0, ra, t0
#endif

#ifdef CONFIG_SMP
        PTE_SC          t0, t1, 0
        beqz            t0, smp_pgtable_change_store
#else
        PTR_S           t0, t1, 0
#endif
        tlbsrch
        PTR_BSTRINS     t1, zero, _PTE_T_LOG2, _PTE_T_LOG2
        PTR_L           t0, t1, 0
        PTR_L           t1, t1, _PTE_T_SIZE
        csrwr           t0, LOONGARCH_CSR_TLBELO0
        csrwr           t1, LOONGARCH_CSR_TLBELO1
        tlbwr

        csrrd           t0, EXCEPTION_KS0
        csrrd           t1, EXCEPTION_KS1
        csrrd           ra, EXCEPTION_KS2
        ertn

vmalloc_store:
        la_abs          t1, swapper_pg_dir
        b               vmalloc_done_store

        /* This is the entry point of a huge page. */
tlb_huge_update_store:
#ifdef CONFIG_SMP
        PTE_LL          ra, t1, 0
#else
        PTR_ROTRI       ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1)
#endif

#ifdef CONFIG_64BIT
        andi            t0, ra, _PAGE_PRESENT | _PAGE_WRITE
        xori            t0, t0, _PAGE_PRESENT | _PAGE_WRITE
#else
        PTR_LI          t0, _PAGE_PRESENT | _PAGE_WRITE
        and             t0, t0, ra
        nor             t0, t0, zero
#endif

        bnez            t0, nopage_tlb_store

#ifdef CONFIG_SMP
        ori             t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
        PTE_SC          t0, t1, 0
        beqz            t0, tlb_huge_update_store
        ori             t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
#else
#ifdef CONFIG_64BIT
        ori             t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
#else
        PTR_LI          t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
        or              t0, ra, t0
#endif
        PTR_S           t0, t1, 0
#endif
        csrrd           ra, LOONGARCH_CSR_ASID
        csrrd           t1, LOONGARCH_CSR_BADV
        andi            ra, ra, CSR_ASID_ASID
        invtlb          INVTLB_ADDR_GFALSE_AND_ASID, ra, t1

        /*
         * A huge PTE describes an area the size of the
         * configured huge page size. This is twice the
         * of the large TLB entry size we intend to use.
         * A TLB entry half the size of the configured
         * huge page size is configured into entrylo0
         * and entrylo1 to cover the contiguous huge PTE
         * address space.
         */
        /* Huge page: Move Global bit */
        xori            t0, t0, _PAGE_HUGE
        lu12i.w         t1, _PAGE_HGLOBAL >> 12
        and             t1, t0, t1
        PTR_SRLI        t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
        or              t0, t0, t1

        move            ra, t0
        csrwr           ra, LOONGARCH_CSR_TLBELO0

        /* Convert to entrylo1 */
        PTR_ADDI        t1, zero, 1
        PTR_SLLI        t1, t1, (HPAGE_SHIFT - 1)
        PTR_ADD         t0, t0, t1
        csrwr           t0, LOONGARCH_CSR_TLBELO1

        /* Set huge page tlb entry size */
        PTR_LI          t0, (CSR_TLBIDX_PS >> 16) << 16
        PTR_LI          t1, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT))
        csrxchg         t1, t0, LOONGARCH_CSR_TLBIDX

        tlbfill

        /* Reset default page size */
        PTR_LI          t0, (CSR_TLBIDX_PS >> 16) << 16
        PTR_LI          t1, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT))
        csrxchg         t1, t0, LOONGARCH_CSR_TLBIDX

        csrrd           t0, EXCEPTION_KS0
        csrrd           t1, EXCEPTION_KS1
        csrrd           ra, EXCEPTION_KS2
        ertn

nopage_tlb_store:
        dbar            0x700
        csrrd           ra, EXCEPTION_KS2
        la_abs          t0, tlb_do_page_fault_1
        jr              t0
SYM_CODE_END(handle_tlb_store)

SYM_CODE_START(handle_tlb_store_ptw)
        UNWIND_HINT_UNDEFINED
        csrwr           t0, LOONGARCH_CSR_KS0
        csrwr           t1, LOONGARCH_CSR_KS1
        la_abs          t0, tlb_do_page_fault_1
        jr              t0
SYM_CODE_END(handle_tlb_store_ptw)

SYM_CODE_START(handle_tlb_modify)
        UNWIND_HINT_UNDEFINED
        csrwr           t0, EXCEPTION_KS0
        csrwr           t1, EXCEPTION_KS1
        csrwr           ra, EXCEPTION_KS2

        /*
         * The vmalloc handling is not in the hotpath.
         */
        csrrd           t0, LOONGARCH_CSR_BADV
        bltz            t0, vmalloc_modify
        csrrd           t1, LOONGARCH_CSR_PGDL

vmalloc_done_modify:
        /* Get PGD offset in bytes */
#ifdef CONFIG_32BIT
        PTR_BSTRPICK    ra, t0, 31, PGDIR_SHIFT
#else
        PTR_BSTRPICK    ra, t0, PTRS_PER_PGD_BITS + PGDIR_SHIFT - 1, PGDIR_SHIFT
#endif
        PTR_ALSL        t1, ra, t1, _PGD_T_LOG2

#if CONFIG_PGTABLE_LEVELS > 3
        PTR_L           t1, t1, 0
        PTR_BSTRPICK    ra, t0, PTRS_PER_PUD_BITS + PUD_SHIFT - 1, PUD_SHIFT
        PTR_ALSL        t1, ra, t1, _PMD_T_LOG2
#endif
#if CONFIG_PGTABLE_LEVELS > 2
        PTR_L           t1, t1, 0
        PTR_BSTRPICK    ra, t0, PTRS_PER_PMD_BITS + PMD_SHIFT - 1, PMD_SHIFT
        PTR_ALSL        t1, ra, t1, _PMD_T_LOG2
#endif
        PTR_L           ra, t1, 0

        /*
         * For huge tlb entries, pmde doesn't contain an address but
         * instead contains the tlb pte. Check the PAGE_HUGE bit and
         * see if we need to jump to huge tlb processing.
         */
        PTR_ROTRI       ra, ra, _PAGE_HUGE_SHIFT + 1
        bltz            ra, tlb_huge_update_modify

        PTR_ROTRI       ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1)
        PTR_BSTRPICK    t0, t0, PTRS_PER_PTE_BITS + PAGE_SHIFT - 1, PAGE_SHIFT
        PTR_ALSL        t1, t0, ra, _PTE_T_LOG2

#ifdef CONFIG_SMP
smp_pgtable_change_modify:
        PTE_LL          t0, t1, 0
#else
        PTR_L           t0, t1, 0
#endif
#ifdef CONFIG_64BIT
        andi            ra, t0, _PAGE_WRITE
#else
        PTR_LI          ra, _PAGE_WRITE
        and             ra, t0, ra
#endif

        beqz            ra, nopage_tlb_modify

#ifdef CONFIG_64BIT
        ori             t0, t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
#else
        PTR_LI          ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
        or              t0, ra, t0
#endif

#ifdef CONFIG_SMP
        PTE_SC          t0, t1, 0
        beqz            t0, smp_pgtable_change_modify
#else
        PTR_S           t0, t1, 0
#endif
        tlbsrch
        PTR_BSTRINS     t1, zero, _PTE_T_LOG2, _PTE_T_LOG2
        PTR_L           t0, t1, 0
        PTR_L           t1, t1, _PTE_T_SIZE
        csrwr           t0, LOONGARCH_CSR_TLBELO0
        csrwr           t1, LOONGARCH_CSR_TLBELO1
        tlbwr

        csrrd           t0, EXCEPTION_KS0
        csrrd           t1, EXCEPTION_KS1
        csrrd           ra, EXCEPTION_KS2
        ertn

vmalloc_modify:
        la_abs          t1, swapper_pg_dir
        b               vmalloc_done_modify

        /* This is the entry point of a huge page. */
tlb_huge_update_modify:
#ifdef CONFIG_SMP
        PTE_LL          ra, t1, 0
#else
        PTR_ROTRI       ra, ra, BITS_PER_LONG - (_PAGE_HUGE_SHIFT + 1)
#endif

#ifdef CONFIG_64BIT
        andi            t0, ra, _PAGE_WRITE
#else
        PTR_LI          t0, _PAGE_WRITE
        and             t0, ra, t0
#endif

        beqz            t0, nopage_tlb_modify

#ifdef CONFIG_SMP
        ori             t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
        PTE_SC          t0, t1, 0
        beqz            t0, tlb_huge_update_modify
        ori             t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
#else
#ifdef CONFIG_64BIT
        ori             t0, ra, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
#else
        PTR_LI          t0, (_PAGE_VALID | _PAGE_DIRTY | _PAGE_MODIFIED)
        or              t0, ra, t0
#endif
        PTR_S           t0, t1, 0
#endif
        csrrd           ra, LOONGARCH_CSR_ASID
        csrrd           t1, LOONGARCH_CSR_BADV
        andi            ra, ra, CSR_ASID_ASID
        invtlb          INVTLB_ADDR_GFALSE_AND_ASID, ra, t1

        /*
         * A huge PTE describes an area the size of the
         * configured huge page size. This is twice the
         * of the large TLB entry size we intend to use.
         * A TLB entry half the size of the configured
         * huge page size is configured into entrylo0
         * and entrylo1 to cover the contiguous huge PTE
         * address space.
         */
        /* Huge page: Move Global bit */
        xori            t0, t0, _PAGE_HUGE
        lu12i.w         t1, _PAGE_HGLOBAL >> 12
        and             t1, t0, t1
        PTR_SRLI        t1, t1, (_PAGE_HGLOBAL_SHIFT - _PAGE_GLOBAL_SHIFT)
        or              t0, t0, t1

        move            ra, t0
        csrwr           ra, LOONGARCH_CSR_TLBELO0

        /* Convert to entrylo1 */
        PTR_ADDI        t1, zero, 1
        PTR_SLLI        t1, t1, (HPAGE_SHIFT - 1)
        PTR_ADD         t0, t0, t1
        csrwr           t0, LOONGARCH_CSR_TLBELO1

        /* Set huge page tlb entry size */
        PTR_LI          t0, (CSR_TLBIDX_PS >> 16) << 16
        PTR_LI          t1, (PS_HUGE_SIZE << (CSR_TLBIDX_PS_SHIFT))
        csrxchg         t1, t0, LOONGARCH_CSR_TLBIDX

        tlbfill

        /* Reset default page size */
        PTR_LI          t0, (CSR_TLBIDX_PS >> 16) << 16
        PTR_LI          t1, (PS_DEFAULT_SIZE << (CSR_TLBIDX_PS_SHIFT))
        csrxchg         t1, t0, LOONGARCH_CSR_TLBIDX

        csrrd           t0, EXCEPTION_KS0
        csrrd           t1, EXCEPTION_KS1
        csrrd           ra, EXCEPTION_KS2
        ertn

nopage_tlb_modify:
        dbar            0x700
        csrrd           ra, EXCEPTION_KS2
        la_abs          t0, tlb_do_page_fault_1
        jr              t0
SYM_CODE_END(handle_tlb_modify)

SYM_CODE_START(handle_tlb_modify_ptw)
        UNWIND_HINT_UNDEFINED
        csrwr           t0, LOONGARCH_CSR_KS0
        csrwr           t1, LOONGARCH_CSR_KS1
        la_abs          t0, tlb_do_page_fault_1
        jr              t0
SYM_CODE_END(handle_tlb_modify_ptw)

#ifdef CONFIG_32BIT
SYM_CODE_START(handle_tlb_refill)
        UNWIND_HINT_UNDEFINED
        csrwr           t0, EXCEPTION_KS0
        csrwr           t1, EXCEPTION_KS1
        csrwr           ra, EXCEPTION_KS2
        li.w            ra, 0x1fffffff

        csrrd           t0, LOONGARCH_CSR_PGD
        csrrd           t1, LOONGARCH_CSR_TLBRBADV
        srli.w          t1, t1, PGDIR_SHIFT
        slli.w          t1, t1, 0x2
        add.w           t0, t0, t1
        and             t0, t0, ra

        ld.w            t0, t0, 0
        csrrd           t1, LOONGARCH_CSR_TLBRBADV
        slli.w          t1, t1, (32 - PGDIR_SHIFT)
        srli.w          t1, t1, (32 - PGDIR_SHIFT + PAGE_SHIFT + 1)
        slli.w          t1, t1, (0x2 + 1)
        add.w           t0, t0, t1
        and             t0, t0, ra

        ld.w            t1, t0, 0x0
        csrwr           t1, LOONGARCH_CSR_TLBRELO0

        ld.w            t1, t0, 0x4
        csrwr           t1, LOONGARCH_CSR_TLBRELO1

        tlbfill
        csrrd           t0, EXCEPTION_KS0
        csrrd           t1, EXCEPTION_KS1
        csrrd           ra, EXCEPTION_KS2
        ertn
SYM_CODE_END(handle_tlb_refill)
#endif

#ifdef CONFIG_64BIT
SYM_CODE_START(handle_tlb_refill)
        UNWIND_HINT_UNDEFINED
        csrwr           t0, LOONGARCH_CSR_TLBRSAVE
        csrrd           t0, LOONGARCH_CSR_PGD
        lddir           t0, t0, 3
#if CONFIG_PGTABLE_LEVELS > 3
        lddir           t0, t0, 2
#endif
#if CONFIG_PGTABLE_LEVELS > 2
        lddir           t0, t0, 1
#endif
        ldpte           t0, 0
        ldpte           t0, 1
        tlbfill
        csrrd           t0, LOONGARCH_CSR_TLBRSAVE
        ertn
SYM_CODE_END(handle_tlb_refill)
#endif