/* SPDX-License-Identifier: GPL-2.0 */
/*
 * Author: Lu Zeng <zenglu@loongson.cn>
 *         Pei Huang <huangpei@loongson.cn>
 *         Huacai Chen <chenhuacai@loongson.cn>
 *
 * Copyright (C) 2020-2022 Loongson Technology Corporation Limited
 */
#include <linux/export.h>
#include <asm/asm.h>
#include <asm/asmmacro.h>
#include <asm/asm-extable.h>
#include <asm/asm-offsets.h>
#include <asm/errno.h>
#include <asm/fpregdef.h>
#include <asm/loongarch.h>
#include <asm/regdef.h>
#include <asm/unwind_hints.h>

#define FPU_REG_WIDTH           8
#define LSX_REG_WIDTH           16
#define LASX_REG_WIDTH          32

        .macro  EX insn, reg, src, offs
.ex\@:  \insn   \reg, \src, \offs
        _asm_extable .ex\@, .L_fpu_fault
        .endm

        .macro sc_save_fp base
        EX      fst.d   $f0,  \base, (0 * FPU_REG_WIDTH)
        EX      fst.d   $f1,  \base, (1 * FPU_REG_WIDTH)
        EX      fst.d   $f2,  \base, (2 * FPU_REG_WIDTH)
        EX      fst.d   $f3,  \base, (3 * FPU_REG_WIDTH)
        EX      fst.d   $f4,  \base, (4 * FPU_REG_WIDTH)
        EX      fst.d   $f5,  \base, (5 * FPU_REG_WIDTH)
        EX      fst.d   $f6,  \base, (6 * FPU_REG_WIDTH)
        EX      fst.d   $f7,  \base, (7 * FPU_REG_WIDTH)
        EX      fst.d   $f8,  \base, (8 * FPU_REG_WIDTH)
        EX      fst.d   $f9,  \base, (9 * FPU_REG_WIDTH)
        EX      fst.d   $f10, \base, (10 * FPU_REG_WIDTH)
        EX      fst.d   $f11, \base, (11 * FPU_REG_WIDTH)
        EX      fst.d   $f12, \base, (12 * FPU_REG_WIDTH)
        EX      fst.d   $f13, \base, (13 * FPU_REG_WIDTH)
        EX      fst.d   $f14, \base, (14 * FPU_REG_WIDTH)
        EX      fst.d   $f15, \base, (15 * FPU_REG_WIDTH)
        EX      fst.d   $f16, \base, (16 * FPU_REG_WIDTH)
        EX      fst.d   $f17, \base, (17 * FPU_REG_WIDTH)
        EX      fst.d   $f18, \base, (18 * FPU_REG_WIDTH)
        EX      fst.d   $f19, \base, (19 * FPU_REG_WIDTH)
        EX      fst.d   $f20, \base, (20 * FPU_REG_WIDTH)
        EX      fst.d   $f21, \base, (21 * FPU_REG_WIDTH)
        EX      fst.d   $f22, \base, (22 * FPU_REG_WIDTH)
        EX      fst.d   $f23, \base, (23 * FPU_REG_WIDTH)
        EX      fst.d   $f24, \base, (24 * FPU_REG_WIDTH)
        EX      fst.d   $f25, \base, (25 * FPU_REG_WIDTH)
        EX      fst.d   $f26, \base, (26 * FPU_REG_WIDTH)
        EX      fst.d   $f27, \base, (27 * FPU_REG_WIDTH)
        EX      fst.d   $f28, \base, (28 * FPU_REG_WIDTH)
        EX      fst.d   $f29, \base, (29 * FPU_REG_WIDTH)
        EX      fst.d   $f30, \base, (30 * FPU_REG_WIDTH)
        EX      fst.d   $f31, \base, (31 * FPU_REG_WIDTH)
        .endm

        .macro sc_restore_fp base
        EX      fld.d   $f0,  \base, (0 * FPU_REG_WIDTH)
        EX      fld.d   $f1,  \base, (1 * FPU_REG_WIDTH)
        EX      fld.d   $f2,  \base, (2 * FPU_REG_WIDTH)
        EX      fld.d   $f3,  \base, (3 * FPU_REG_WIDTH)
        EX      fld.d   $f4,  \base, (4 * FPU_REG_WIDTH)
        EX      fld.d   $f5,  \base, (5 * FPU_REG_WIDTH)
        EX      fld.d   $f6,  \base, (6 * FPU_REG_WIDTH)
        EX      fld.d   $f7,  \base, (7 * FPU_REG_WIDTH)
        EX      fld.d   $f8,  \base, (8 * FPU_REG_WIDTH)
        EX      fld.d   $f9,  \base, (9 * FPU_REG_WIDTH)
        EX      fld.d   $f10, \base, (10 * FPU_REG_WIDTH)
        EX      fld.d   $f11, \base, (11 * FPU_REG_WIDTH)
        EX      fld.d   $f12, \base, (12 * FPU_REG_WIDTH)
        EX      fld.d   $f13, \base, (13 * FPU_REG_WIDTH)
        EX      fld.d   $f14, \base, (14 * FPU_REG_WIDTH)
        EX      fld.d   $f15, \base, (15 * FPU_REG_WIDTH)
        EX      fld.d   $f16, \base, (16 * FPU_REG_WIDTH)
        EX      fld.d   $f17, \base, (17 * FPU_REG_WIDTH)
        EX      fld.d   $f18, \base, (18 * FPU_REG_WIDTH)
        EX      fld.d   $f19, \base, (19 * FPU_REG_WIDTH)
        EX      fld.d   $f20, \base, (20 * FPU_REG_WIDTH)
        EX      fld.d   $f21, \base, (21 * FPU_REG_WIDTH)
        EX      fld.d   $f22, \base, (22 * FPU_REG_WIDTH)
        EX      fld.d   $f23, \base, (23 * FPU_REG_WIDTH)
        EX      fld.d   $f24, \base, (24 * FPU_REG_WIDTH)
        EX      fld.d   $f25, \base, (25 * FPU_REG_WIDTH)
        EX      fld.d   $f26, \base, (26 * FPU_REG_WIDTH)
        EX      fld.d   $f27, \base, (27 * FPU_REG_WIDTH)
        EX      fld.d   $f28, \base, (28 * FPU_REG_WIDTH)
        EX      fld.d   $f29, \base, (29 * FPU_REG_WIDTH)
        EX      fld.d   $f30, \base, (30 * FPU_REG_WIDTH)
        EX      fld.d   $f31, \base, (31 * FPU_REG_WIDTH)
        .endm

#ifdef CONFIG_32BIT
        .macro sc_save_fcc thread tmp0 tmp1
        movcf2gr        \tmp0, $fcc0
        move            \tmp1, \tmp0
        movcf2gr        \tmp0, $fcc1
        bstrins.w       \tmp1, \tmp0, 15, 8
        movcf2gr        \tmp0, $fcc2
        bstrins.w       \tmp1, \tmp0, 23, 16
        movcf2gr        \tmp0, $fcc3
        bstrins.w       \tmp1, \tmp0, 31, 24
        EX      st.w    \tmp1, \thread, THREAD_FCC
        movcf2gr        \tmp0, $fcc4
        move            \tmp1, \tmp0
        movcf2gr        \tmp0, $fcc5
        bstrins.w       \tmp1, \tmp0, 15, 8
        movcf2gr        \tmp0, $fcc6
        bstrins.w       \tmp1, \tmp0, 23, 16
        movcf2gr        \tmp0, $fcc7
        bstrins.w       \tmp1, \tmp0, 31, 24
        EX      st.w    \tmp1, \thread, (THREAD_FCC + 4)
        .endm

        .macro sc_restore_fcc thread tmp0 tmp1
        EX      ld.w    \tmp0, \thread, THREAD_FCC
        bstrpick.w      \tmp1, \tmp0, 7, 0
        movgr2cf        $fcc0, \tmp1
        bstrpick.w      \tmp1, \tmp0, 15, 8
        movgr2cf        $fcc1, \tmp1
        bstrpick.w      \tmp1, \tmp0, 23, 16
        movgr2cf        $fcc2, \tmp1
        bstrpick.w      \tmp1, \tmp0, 31, 24
        movgr2cf        $fcc3, \tmp1
        EX      ld.w    \tmp0, \thread, (THREAD_FCC + 4)
        bstrpick.w      \tmp1, \tmp0, 7, 0
        movgr2cf        $fcc4, \tmp1
        bstrpick.w      \tmp1, \tmp0, 15, 8
        movgr2cf        $fcc5, \tmp1
        bstrpick.w      \tmp1, \tmp0, 23, 16
        movgr2cf        $fcc6, \tmp1
        bstrpick.w      \tmp1, \tmp0, 31, 24
        movgr2cf        $fcc7, \tmp1
        .endm
#else
        .macro sc_save_fcc base, tmp0, tmp1
        movcf2gr        \tmp0, $fcc0
        move            \tmp1, \tmp0
        movcf2gr        \tmp0, $fcc1
        bstrins.d       \tmp1, \tmp0, 15, 8
        movcf2gr        \tmp0, $fcc2
        bstrins.d       \tmp1, \tmp0, 23, 16
        movcf2gr        \tmp0, $fcc3
        bstrins.d       \tmp1, \tmp0, 31, 24
        movcf2gr        \tmp0, $fcc4
        bstrins.d       \tmp1, \tmp0, 39, 32
        movcf2gr        \tmp0, $fcc5
        bstrins.d       \tmp1, \tmp0, 47, 40
        movcf2gr        \tmp0, $fcc6
        bstrins.d       \tmp1, \tmp0, 55, 48
        movcf2gr        \tmp0, $fcc7
        bstrins.d       \tmp1, \tmp0, 63, 56
        EX      st.d    \tmp1, \base, 0
        .endm

        .macro sc_restore_fcc base, tmp0, tmp1
        EX      ld.d    \tmp0, \base, 0
        bstrpick.d      \tmp1, \tmp0, 7, 0
        movgr2cf        $fcc0, \tmp1
        bstrpick.d      \tmp1, \tmp0, 15, 8
        movgr2cf        $fcc1, \tmp1
        bstrpick.d      \tmp1, \tmp0, 23, 16
        movgr2cf        $fcc2, \tmp1
        bstrpick.d      \tmp1, \tmp0, 31, 24
        movgr2cf        $fcc3, \tmp1
        bstrpick.d      \tmp1, \tmp0, 39, 32
        movgr2cf        $fcc4, \tmp1
        bstrpick.d      \tmp1, \tmp0, 47, 40
        movgr2cf        $fcc5, \tmp1
        bstrpick.d      \tmp1, \tmp0, 55, 48
        movgr2cf        $fcc6, \tmp1
        bstrpick.d      \tmp1, \tmp0, 63, 56
        movgr2cf        $fcc7, \tmp1
        .endm
#endif

        .macro sc_save_fcsr base, tmp0
        movfcsr2gr      \tmp0, fcsr0
        EX      st.w    \tmp0, \base, 0
#if defined(CONFIG_CPU_HAS_LBT)
        /* TM bit is always 0 if LBT not supported */
        andi            \tmp0, \tmp0, FPU_CSR_TM
        beqz            \tmp0, 1f
        x86clrtm
1:
#endif
        .endm

        .macro sc_restore_fcsr base, tmp0
        EX      ld.w    \tmp0, \base, 0
        movgr2fcsr      fcsr0, \tmp0
        .endm

        .macro sc_save_lsx base
#ifdef CONFIG_CPU_HAS_LSX
        EX      vst     $vr0,  \base, (0 * LSX_REG_WIDTH)
        EX      vst     $vr1,  \base, (1 * LSX_REG_WIDTH)
        EX      vst     $vr2,  \base, (2 * LSX_REG_WIDTH)
        EX      vst     $vr3,  \base, (3 * LSX_REG_WIDTH)
        EX      vst     $vr4,  \base, (4 * LSX_REG_WIDTH)
        EX      vst     $vr5,  \base, (5 * LSX_REG_WIDTH)
        EX      vst     $vr6,  \base, (6 * LSX_REG_WIDTH)
        EX      vst     $vr7,  \base, (7 * LSX_REG_WIDTH)
        EX      vst     $vr8,  \base, (8 * LSX_REG_WIDTH)
        EX      vst     $vr9,  \base, (9 * LSX_REG_WIDTH)
        EX      vst     $vr10, \base, (10 * LSX_REG_WIDTH)
        EX      vst     $vr11, \base, (11 * LSX_REG_WIDTH)
        EX      vst     $vr12, \base, (12 * LSX_REG_WIDTH)
        EX      vst     $vr13, \base, (13 * LSX_REG_WIDTH)
        EX      vst     $vr14, \base, (14 * LSX_REG_WIDTH)
        EX      vst     $vr15, \base, (15 * LSX_REG_WIDTH)
        EX      vst     $vr16, \base, (16 * LSX_REG_WIDTH)
        EX      vst     $vr17, \base, (17 * LSX_REG_WIDTH)
        EX      vst     $vr18, \base, (18 * LSX_REG_WIDTH)
        EX      vst     $vr19, \base, (19 * LSX_REG_WIDTH)
        EX      vst     $vr20, \base, (20 * LSX_REG_WIDTH)
        EX      vst     $vr21, \base, (21 * LSX_REG_WIDTH)
        EX      vst     $vr22, \base, (22 * LSX_REG_WIDTH)
        EX      vst     $vr23, \base, (23 * LSX_REG_WIDTH)
        EX      vst     $vr24, \base, (24 * LSX_REG_WIDTH)
        EX      vst     $vr25, \base, (25 * LSX_REG_WIDTH)
        EX      vst     $vr26, \base, (26 * LSX_REG_WIDTH)
        EX      vst     $vr27, \base, (27 * LSX_REG_WIDTH)
        EX      vst     $vr28, \base, (28 * LSX_REG_WIDTH)
        EX      vst     $vr29, \base, (29 * LSX_REG_WIDTH)
        EX      vst     $vr30, \base, (30 * LSX_REG_WIDTH)
        EX      vst     $vr31, \base, (31 * LSX_REG_WIDTH)
#endif
        .endm

        .macro sc_restore_lsx base
#ifdef CONFIG_CPU_HAS_LSX
        EX      vld     $vr0,  \base, (0 * LSX_REG_WIDTH)
        EX      vld     $vr1,  \base, (1 * LSX_REG_WIDTH)
        EX      vld     $vr2,  \base, (2 * LSX_REG_WIDTH)
        EX      vld     $vr3,  \base, (3 * LSX_REG_WIDTH)
        EX      vld     $vr4,  \base, (4 * LSX_REG_WIDTH)
        EX      vld     $vr5,  \base, (5 * LSX_REG_WIDTH)
        EX      vld     $vr6,  \base, (6 * LSX_REG_WIDTH)
        EX      vld     $vr7,  \base, (7 * LSX_REG_WIDTH)
        EX      vld     $vr8,  \base, (8 * LSX_REG_WIDTH)
        EX      vld     $vr9,  \base, (9 * LSX_REG_WIDTH)
        EX      vld     $vr10, \base, (10 * LSX_REG_WIDTH)
        EX      vld     $vr11, \base, (11 * LSX_REG_WIDTH)
        EX      vld     $vr12, \base, (12 * LSX_REG_WIDTH)
        EX      vld     $vr13, \base, (13 * LSX_REG_WIDTH)
        EX      vld     $vr14, \base, (14 * LSX_REG_WIDTH)
        EX      vld     $vr15, \base, (15 * LSX_REG_WIDTH)
        EX      vld     $vr16, \base, (16 * LSX_REG_WIDTH)
        EX      vld     $vr17, \base, (17 * LSX_REG_WIDTH)
        EX      vld     $vr18, \base, (18 * LSX_REG_WIDTH)
        EX      vld     $vr19, \base, (19 * LSX_REG_WIDTH)
        EX      vld     $vr20, \base, (20 * LSX_REG_WIDTH)
        EX      vld     $vr21, \base, (21 * LSX_REG_WIDTH)
        EX      vld     $vr22, \base, (22 * LSX_REG_WIDTH)
        EX      vld     $vr23, \base, (23 * LSX_REG_WIDTH)
        EX      vld     $vr24, \base, (24 * LSX_REG_WIDTH)
        EX      vld     $vr25, \base, (25 * LSX_REG_WIDTH)
        EX      vld     $vr26, \base, (26 * LSX_REG_WIDTH)
        EX      vld     $vr27, \base, (27 * LSX_REG_WIDTH)
        EX      vld     $vr28, \base, (28 * LSX_REG_WIDTH)
        EX      vld     $vr29, \base, (29 * LSX_REG_WIDTH)
        EX      vld     $vr30, \base, (30 * LSX_REG_WIDTH)
        EX      vld     $vr31, \base, (31 * LSX_REG_WIDTH)
#endif
        .endm

        .macro sc_save_lasx base
#ifdef CONFIG_CPU_HAS_LASX
        EX      xvst    $xr0,  \base, (0 * LASX_REG_WIDTH)
        EX      xvst    $xr1,  \base, (1 * LASX_REG_WIDTH)
        EX      xvst    $xr2,  \base, (2 * LASX_REG_WIDTH)
        EX      xvst    $xr3,  \base, (3 * LASX_REG_WIDTH)
        EX      xvst    $xr4,  \base, (4 * LASX_REG_WIDTH)
        EX      xvst    $xr5,  \base, (5 * LASX_REG_WIDTH)
        EX      xvst    $xr6,  \base, (6 * LASX_REG_WIDTH)
        EX      xvst    $xr7,  \base, (7 * LASX_REG_WIDTH)
        EX      xvst    $xr8,  \base, (8 * LASX_REG_WIDTH)
        EX      xvst    $xr9,  \base, (9 * LASX_REG_WIDTH)
        EX      xvst    $xr10, \base, (10 * LASX_REG_WIDTH)
        EX      xvst    $xr11, \base, (11 * LASX_REG_WIDTH)
        EX      xvst    $xr12, \base, (12 * LASX_REG_WIDTH)
        EX      xvst    $xr13, \base, (13 * LASX_REG_WIDTH)
        EX      xvst    $xr14, \base, (14 * LASX_REG_WIDTH)
        EX      xvst    $xr15, \base, (15 * LASX_REG_WIDTH)
        EX      xvst    $xr16, \base, (16 * LASX_REG_WIDTH)
        EX      xvst    $xr17, \base, (17 * LASX_REG_WIDTH)
        EX      xvst    $xr18, \base, (18 * LASX_REG_WIDTH)
        EX      xvst    $xr19, \base, (19 * LASX_REG_WIDTH)
        EX      xvst    $xr20, \base, (20 * LASX_REG_WIDTH)
        EX      xvst    $xr21, \base, (21 * LASX_REG_WIDTH)
        EX      xvst    $xr22, \base, (22 * LASX_REG_WIDTH)
        EX      xvst    $xr23, \base, (23 * LASX_REG_WIDTH)
        EX      xvst    $xr24, \base, (24 * LASX_REG_WIDTH)
        EX      xvst    $xr25, \base, (25 * LASX_REG_WIDTH)
        EX      xvst    $xr26, \base, (26 * LASX_REG_WIDTH)
        EX      xvst    $xr27, \base, (27 * LASX_REG_WIDTH)
        EX      xvst    $xr28, \base, (28 * LASX_REG_WIDTH)
        EX      xvst    $xr29, \base, (29 * LASX_REG_WIDTH)
        EX      xvst    $xr30, \base, (30 * LASX_REG_WIDTH)
        EX      xvst    $xr31, \base, (31 * LASX_REG_WIDTH)
#endif
        .endm

        .macro sc_restore_lasx base
#ifdef CONFIG_CPU_HAS_LASX
        EX      xvld    $xr0,  \base, (0 * LASX_REG_WIDTH)
        EX      xvld    $xr1,  \base, (1 * LASX_REG_WIDTH)
        EX      xvld    $xr2,  \base, (2 * LASX_REG_WIDTH)
        EX      xvld    $xr3,  \base, (3 * LASX_REG_WIDTH)
        EX      xvld    $xr4,  \base, (4 * LASX_REG_WIDTH)
        EX      xvld    $xr5,  \base, (5 * LASX_REG_WIDTH)
        EX      xvld    $xr6,  \base, (6 * LASX_REG_WIDTH)
        EX      xvld    $xr7,  \base, (7 * LASX_REG_WIDTH)
        EX      xvld    $xr8,  \base, (8 * LASX_REG_WIDTH)
        EX      xvld    $xr9,  \base, (9 * LASX_REG_WIDTH)
        EX      xvld    $xr10, \base, (10 * LASX_REG_WIDTH)
        EX      xvld    $xr11, \base, (11 * LASX_REG_WIDTH)
        EX      xvld    $xr12, \base, (12 * LASX_REG_WIDTH)
        EX      xvld    $xr13, \base, (13 * LASX_REG_WIDTH)
        EX      xvld    $xr14, \base, (14 * LASX_REG_WIDTH)
        EX      xvld    $xr15, \base, (15 * LASX_REG_WIDTH)
        EX      xvld    $xr16, \base, (16 * LASX_REG_WIDTH)
        EX      xvld    $xr17, \base, (17 * LASX_REG_WIDTH)
        EX      xvld    $xr18, \base, (18 * LASX_REG_WIDTH)
        EX      xvld    $xr19, \base, (19 * LASX_REG_WIDTH)
        EX      xvld    $xr20, \base, (20 * LASX_REG_WIDTH)
        EX      xvld    $xr21, \base, (21 * LASX_REG_WIDTH)
        EX      xvld    $xr22, \base, (22 * LASX_REG_WIDTH)
        EX      xvld    $xr23, \base, (23 * LASX_REG_WIDTH)
        EX      xvld    $xr24, \base, (24 * LASX_REG_WIDTH)
        EX      xvld    $xr25, \base, (25 * LASX_REG_WIDTH)
        EX      xvld    $xr26, \base, (26 * LASX_REG_WIDTH)
        EX      xvld    $xr27, \base, (27 * LASX_REG_WIDTH)
        EX      xvld    $xr28, \base, (28 * LASX_REG_WIDTH)
        EX      xvld    $xr29, \base, (29 * LASX_REG_WIDTH)
        EX      xvld    $xr30, \base, (30 * LASX_REG_WIDTH)
        EX      xvld    $xr31, \base, (31 * LASX_REG_WIDTH)
#endif
        .endm

/*
 * Save a thread's fp context.
 */
SYM_FUNC_START(_save_fp)
        fpu_save_csr    a0 t1
        fpu_save_double a0 t1                   # clobbers t1
        fpu_save_cc     a0 t1 t2                # clobbers t1, t2
        jr              ra
SYM_FUNC_END(_save_fp)
EXPORT_SYMBOL(_save_fp)

/*
 * Restore a thread's fp context.
 */
SYM_FUNC_START(_restore_fp)
        fpu_restore_double      a0 t1           # clobbers t1
        fpu_restore_csr         a0 t1 t2
        fpu_restore_cc          a0 t1 t2        # clobbers t1, t2
        jr                      ra
SYM_FUNC_END(_restore_fp)

#ifdef CONFIG_CPU_HAS_LSX

/*
 * Save a thread's LSX vector context.
 */
SYM_FUNC_START(_save_lsx)
        lsx_save_all    a0 t1 t2
        jr      ra
SYM_FUNC_END(_save_lsx)
EXPORT_SYMBOL(_save_lsx)

/*
 * Restore a thread's LSX vector context.
 */
SYM_FUNC_START(_restore_lsx)
        lsx_restore_all a0 t1 t2
        jr      ra
SYM_FUNC_END(_restore_lsx)

SYM_FUNC_START(_save_lsx_upper)
        lsx_save_all_upper a0 t0 t1
        jr      ra
SYM_FUNC_END(_save_lsx_upper)

SYM_FUNC_START(_restore_lsx_upper)
        lsx_restore_all_upper a0 t0 t1
        jr      ra
SYM_FUNC_END(_restore_lsx_upper)
EXPORT_SYMBOL(_restore_lsx_upper)

SYM_FUNC_START(_init_lsx_upper)
        lsx_init_all_upper t1
        jr      ra
SYM_FUNC_END(_init_lsx_upper)
#endif

#ifdef CONFIG_CPU_HAS_LASX

/*
 * Save a thread's LASX vector context.
 */
SYM_FUNC_START(_save_lasx)
        lasx_save_all   a0 t1 t2
        jr      ra
SYM_FUNC_END(_save_lasx)
EXPORT_SYMBOL(_save_lasx)

/*
 * Restore a thread's LASX vector context.
 */
SYM_FUNC_START(_restore_lasx)
        lasx_restore_all a0 t1 t2
        jr      ra
SYM_FUNC_END(_restore_lasx)

SYM_FUNC_START(_save_lasx_upper)
        lasx_save_all_upper a0 t0 t1
        jr      ra
SYM_FUNC_END(_save_lasx_upper)

SYM_FUNC_START(_restore_lasx_upper)
        lasx_restore_all_upper a0 t0 t1
        jr      ra
SYM_FUNC_END(_restore_lasx_upper)
EXPORT_SYMBOL(_restore_lasx_upper)

SYM_FUNC_START(_init_lasx_upper)
        lasx_init_all_upper t1
        jr      ra
SYM_FUNC_END(_init_lasx_upper)
#endif

/*
 * Load the FPU with signalling NANS.  This bit pattern we're using has
 * the property that no matter whether considered as single or as double
 * precision represents signaling NANS.
 *
 * The value to initialize fcsr0 to comes in $a0.
 */

SYM_FUNC_START(_init_fpu)
        li.w    t1, CSR_EUEN_FPEN
        csrxchg t1, t1, LOONGARCH_CSR_EUEN

        movgr2fcsr      fcsr0, a0

        li.w    t1, -1                          # SNaN

#ifdef CONFIG_32BIT
        movgr2fr.w      $f0, t1
        movgr2frh.w     $f0, t1
        movgr2fr.w      $f1, t1
        movgr2frh.w     $f1, t1
        movgr2fr.w      $f2, t1
        movgr2frh.w     $f2, t1
        movgr2fr.w      $f3, t1
        movgr2frh.w     $f3, t1
        movgr2fr.w      $f4, t1
        movgr2frh.w     $f4, t1
        movgr2fr.w      $f5, t1
        movgr2frh.w     $f5, t1
        movgr2fr.w      $f6, t1
        movgr2frh.w     $f6, t1
        movgr2fr.w      $f7, t1
        movgr2frh.w     $f7, t1
        movgr2fr.w      $f8, t1
        movgr2frh.w     $f8, t1
        movgr2fr.w      $f9, t1
        movgr2frh.w     $f9, t1
        movgr2fr.w      $f10, t1
        movgr2frh.w     $f10, t1
        movgr2fr.w      $f11, t1
        movgr2frh.w     $f11, t1
        movgr2fr.w      $f12, t1
        movgr2frh.w     $f12, t1
        movgr2fr.w      $f13, t1
        movgr2frh.w     $f13, t1
        movgr2fr.w      $f14, t1
        movgr2frh.w     $f14, t1
        movgr2fr.w      $f15, t1
        movgr2frh.w     $f15, t1
        movgr2fr.w      $f16, t1
        movgr2frh.w     $f16, t1
        movgr2fr.w      $f17, t1
        movgr2frh.w     $f17, t1
        movgr2fr.w      $f18, t1
        movgr2frh.w     $f18, t1
        movgr2fr.w      $f19, t1
        movgr2frh.w     $f19, t1
        movgr2fr.w      $f20, t1
        movgr2frh.w     $f20, t1
        movgr2fr.w      $f21, t1
        movgr2frh.w     $f21, t1
        movgr2fr.w      $f22, t1
        movgr2frh.w     $f22, t1
        movgr2fr.w      $f23, t1
        movgr2frh.w     $f23, t1
        movgr2fr.w      $f24, t1
        movgr2frh.w     $f24, t1
        movgr2fr.w      $f25, t1
        movgr2frh.w     $f25, t1
        movgr2fr.w      $f26, t1
        movgr2frh.w     $f26, t1
        movgr2fr.w      $f27, t1
        movgr2frh.w     $f27, t1
        movgr2fr.w      $f28, t1
        movgr2frh.w     $f28, t1
        movgr2fr.w      $f29, t1
        movgr2frh.w     $f29, t1
        movgr2fr.w      $f30, t1
        movgr2frh.w     $f30, t1
        movgr2fr.w      $f31, t1
        movgr2frh.w     $f31, t1
#else
        movgr2fr.d      $f0, t1
        movgr2fr.d      $f1, t1
        movgr2fr.d      $f2, t1
        movgr2fr.d      $f3, t1
        movgr2fr.d      $f4, t1
        movgr2fr.d      $f5, t1
        movgr2fr.d      $f6, t1
        movgr2fr.d      $f7, t1
        movgr2fr.d      $f8, t1
        movgr2fr.d      $f9, t1
        movgr2fr.d      $f10, t1
        movgr2fr.d      $f11, t1
        movgr2fr.d      $f12, t1
        movgr2fr.d      $f13, t1
        movgr2fr.d      $f14, t1
        movgr2fr.d      $f15, t1
        movgr2fr.d      $f16, t1
        movgr2fr.d      $f17, t1
        movgr2fr.d      $f18, t1
        movgr2fr.d      $f19, t1
        movgr2fr.d      $f20, t1
        movgr2fr.d      $f21, t1
        movgr2fr.d      $f22, t1
        movgr2fr.d      $f23, t1
        movgr2fr.d      $f24, t1
        movgr2fr.d      $f25, t1
        movgr2fr.d      $f26, t1
        movgr2fr.d      $f27, t1
        movgr2fr.d      $f28, t1
        movgr2fr.d      $f29, t1
        movgr2fr.d      $f30, t1
        movgr2fr.d      $f31, t1
#endif

        jr      ra
SYM_FUNC_END(_init_fpu)

/*
 * a0: fpregs
 * a1: fcc
 * a2: fcsr
 */
SYM_FUNC_START(_save_fp_context)
        sc_save_fcc     a1 t1 t2
        sc_save_fcsr    a2 t1
        sc_save_fp      a0
        li.w            a0, 0                           # success
        jr              ra
SYM_FUNC_END(_save_fp_context)
EXPORT_SYMBOL_GPL(_save_fp_context)

/*
 * a0: fpregs
 * a1: fcc
 * a2: fcsr
 */
SYM_FUNC_START(_restore_fp_context)
        sc_restore_fp   a0
        sc_restore_fcc  a1 t1 t2
        sc_restore_fcsr a2 t1
        li.w            a0, 0                           # success
        jr              ra
SYM_FUNC_END(_restore_fp_context)
EXPORT_SYMBOL_GPL(_restore_fp_context)

/*
 * a0: fpregs
 * a1: fcc
 * a2: fcsr
 */
SYM_FUNC_START(_save_lsx_context)
        sc_save_fcc a1, t0, t1
        sc_save_fcsr a2, t0
        sc_save_lsx a0
        li.w    a0, 0                                   # success
        jr      ra
SYM_FUNC_END(_save_lsx_context)
EXPORT_SYMBOL_GPL(_save_lsx_context)

/*
 * a0: fpregs
 * a1: fcc
 * a2: fcsr
 */
SYM_FUNC_START(_restore_lsx_context)
        sc_restore_lsx a0
        sc_restore_fcc a1, t1, t2
        sc_restore_fcsr a2, t1
        li.w    a0, 0                                   # success
        jr      ra
SYM_FUNC_END(_restore_lsx_context)
EXPORT_SYMBOL_GPL(_restore_lsx_context)

/*
 * a0: fpregs
 * a1: fcc
 * a2: fcsr
 */
SYM_FUNC_START(_save_lasx_context)
        sc_save_fcc a1, t0, t1
        sc_save_fcsr a2, t0
        sc_save_lasx a0
        li.w    a0, 0                                   # success
        jr      ra
SYM_FUNC_END(_save_lasx_context)
EXPORT_SYMBOL_GPL(_save_lasx_context)

/*
 * a0: fpregs
 * a1: fcc
 * a2: fcsr
 */
SYM_FUNC_START(_restore_lasx_context)
        sc_restore_lasx a0
        sc_restore_fcc a1, t1, t2
        sc_restore_fcsr a2, t1
        li.w    a0, 0                                   # success
        jr      ra
SYM_FUNC_END(_restore_lasx_context)
EXPORT_SYMBOL_GPL(_restore_lasx_context)

.L_fpu_fault:
        li.w    a0, -EFAULT                             # failure
        jr      ra

#ifdef CONFIG_CPU_HAS_LBT
STACK_FRAME_NON_STANDARD _restore_fp
#ifdef CONFIG_CPU_HAS_LSX
STACK_FRAME_NON_STANDARD _restore_lsx
#endif
#ifdef CONFIG_CPU_HAS_LASX
STACK_FRAME_NON_STANDARD _restore_lasx
#endif
#endif