/* $NetBSD: fpu.c,v 1.13 2022/08/20 11:34:08 riastradh Exp $ */

/*-
 * Copyright (c) 2014 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Matt Thomas of 3am Software Foundry.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>

__KERNEL_RCSID(1, "$NetBSD: fpu.c,v 1.13 2022/08/20 11:34:08 riastradh Exp $");

#include <sys/param.h>
#include <sys/types.h>
#include <sys/cpu.h>
#include <sys/kernel.h>
#include <sys/kthread.h>
#include <sys/lwp.h>
#include <sys/evcnt.h>

#include <arm/cpufunc.h>
#include <arm/fpu.h>
#include <arm/cpufunc.h>

#include <aarch64/locore.h>
#include <aarch64/reg.h>
#include <aarch64/pcb.h>
#include <aarch64/armreg.h>
#include <aarch64/machdep.h>

static void fpu_state_load(lwp_t *, unsigned int);
static void fpu_state_save(lwp_t *);
static void fpu_state_release(lwp_t *);

const pcu_ops_t pcu_fpu_ops = {
        .pcu_id = PCU_FPU,
        .pcu_state_load = fpu_state_load,
        .pcu_state_save = fpu_state_save,
        .pcu_state_release = fpu_state_release
};

void
fpu_attach(struct cpu_info *ci)
{
        evcnt_attach_dynamic(&ci->ci_vfp_use, EVCNT_TYPE_MISC, NULL,
            ci->ci_cpuname, "vfp use");
        evcnt_attach_dynamic(&ci->ci_vfp_reuse, EVCNT_TYPE_MISC, NULL,
            ci->ci_cpuname, "vfp reuse");
        evcnt_attach_dynamic(&ci->ci_vfp_save, EVCNT_TYPE_MISC, NULL,
            ci->ci_cpuname, "vfp save");
        evcnt_attach_dynamic(&ci->ci_vfp_release, EVCNT_TYPE_MISC, NULL,
            ci->ci_cpuname, "vfp release");
}

static void
fpu_state_load(lwp_t *l, unsigned int flags)
{
        struct pcb * const pcb = lwp_getpcb(l);

        KASSERT(l == curlwp);

        if (__predict_false((flags & PCU_VALID) == 0)) {
                uint64_t mvfr1 = reg_mvfr1_el1_read();
                bool fp16 = false;
                uint32_t fpcr = 0;

                /*
                 * Determine whether ARMv8.2-FP16 binary16
                 * floating-point arithmetic is supported.
                 */
                switch (__SHIFTOUT(mvfr1, MVFR1_FPHP)) {
                case MVFR1_FPHP_HALF_ARITH:
                        fp16 = true;
                        break;
                }

                /* Rounding mode: round to nearest, ties to even.  */
                fpcr |= __SHIFTIN(FPCR_RN, FPCR_RMODE);

                /* NaN propagation or default NaN.   */
                switch (__SHIFTOUT(mvfr1, MVFR1_FPDNAN)) {
                case MVFR1_FPDNAN_NAN:
                        /*
                         * IEEE 754 NaN propagation supported.  Don't
                         * enable default NaN mode.
                         */
                        break;
                default:
                        /*
                         * IEEE 754 NaN propagation not supported, so
                         * enable default NaN mode.
                         */
                        fpcr |= FPCR_DN;
                }

                /* Subnormal arithmetic or flush-to-zero.  */
                switch (__SHIFTOUT(mvfr1, MVFR1_FPFTZ)) {
                case MVFR1_FPFTZ_DENORMAL:
                        /*
                         * IEEE 754 subnormal arithmetic supported.
                         * Don't enable flush-to-zero mode.
                         */
                        break;
                default:
                        /*
                         * IEEE 754 subnormal arithmetic not supported,
                         * so enable flush-to-zero mode.  If FP16 is
                         * supported, also enable flush-to-zero for
                         * binary16 arithmetic.
                         */
                        fpcr |= FPCR_FZ;
                        if (fp16)
                                fpcr |= FPCR_FZ16;
                }

                /* initialize fpregs */
                memset(&pcb->pcb_fpregs, 0, sizeof(pcb->pcb_fpregs));
                pcb->pcb_fpregs.fpcr = fpcr;

                curcpu()->ci_vfp_use.ev_count++;
        } else {
                curcpu()->ci_vfp_reuse.ev_count++;
        }

        /* allow user process to use FP */
        l->l_md.md_cpacr = CPACR_FPEN_ALL;
        reg_cpacr_el1_write(CPACR_FPEN_ALL);
        isb();

        if ((flags & PCU_REENABLE) == 0)
                load_fpregs(&pcb->pcb_fpregs);
}

static void
fpu_state_save(lwp_t *l)
{
        struct pcb * const pcb = lwp_getpcb(l);

        curcpu()->ci_vfp_save.ev_count++;

        reg_cpacr_el1_write(CPACR_FPEN_EL1);    /* fpreg access enable */
        isb();

        save_fpregs(&pcb->pcb_fpregs);

        reg_cpacr_el1_write(CPACR_FPEN_NONE);   /* fpreg access disable */
        isb();
}

static void
fpu_state_release(lwp_t *l)
{
        curcpu()->ci_vfp_release.ev_count++;

        /* disallow user process to use FP */
        l->l_md.md_cpacr = CPACR_FPEN_NONE;
        reg_cpacr_el1_write(CPACR_FPEN_NONE);
        isb();
}

static const struct fpreg zero_fpreg;

/*
 * True if this is a system thread with its own private FPU state.
 */
static inline bool
lwp_system_fpu_p(struct lwp *l)
{

        return (l->l_flag & (LW_SYSTEM|LW_SYSTEM_FPU)) ==
            (LW_SYSTEM|LW_SYSTEM_FPU);
}

void
fpu_kern_enter(void)
{
        struct cpu_info *ci;
        int s;

        if (lwp_system_fpu_p(curlwp) && !cpu_intr_p()) {
                KASSERT(!cpu_softintr_p());
                return;
        }

        /*
         * Block interrupts up to IPL_VM.  We must block preemption
         * since -- if this is a user thread -- there is nowhere to
         * save the kernel fpu state, and if we want this to be usable
         * in interrupts, we can't let interrupts interfere with the
         * fpu state in use since there's nowhere for them to save it.
         */
        s = splvm();
        ci = curcpu();
#if 0
        /*
         * Can't assert this because if the caller holds a spin lock at
         * IPL_VM, and previously held and released a spin lock at
         * higher IPL, the IPL remains raised above IPL_VM.
         */
        KASSERTMSG(ci->ci_cpl <= IPL_VM || cold, "cpl=%d", ci->ci_cpl);
#endif
        KASSERT(ci->ci_kfpu_spl == -1);
        ci->ci_kfpu_spl = s;

        /* Save any fpu state on the current CPU.  */
        pcu_save_all_on_cpu();

        /*
         * Enable the fpu, and wait until it is enabled before
         * executing any further instructions.
         */
        reg_cpacr_el1_write(CPACR_FPEN_ALL);
        isb();
}

void
fpu_kern_leave(void)
{
        struct cpu_info *ci;
        int s;

        if (lwp_system_fpu_p(curlwp) && !cpu_intr_p()) {
                KASSERT(!cpu_softintr_p());
                return;
        }

        ci = curcpu();

#if 0
        /*
         * Can't assert this because if the caller holds a spin lock at
         * IPL_VM, and previously held and released a spin lock at
         * higher IPL, the IPL remains raised above IPL_VM.
         */
        KASSERT(ci->ci_cpl == IPL_VM || cold);
#endif
        KASSERT(ci->ci_kfpu_spl != -1);

        /*
         * Zero the fpu registers; otherwise we might leak secrets
         * through Spectre-class attacks to userland, even if there are
         * no bugs in fpu state management.
         */
        load_fpregs(&zero_fpreg);

        /*
         * Disable the fpu so that the kernel can't accidentally use
         * it again.
         */
        reg_cpacr_el1_write(CPACR_FPEN_NONE);
        isb();

        s = ci->ci_kfpu_spl;
        ci->ci_kfpu_spl = -1;
        splx(s);
}

void
kthread_fpu_enter_md(void)
{

        fpu_load(curlwp);
}

void
kthread_fpu_exit_md(void)
{

        /* XXX Should fpu_state_release zero the registers itself?  */
        load_fpregs(&zero_fpreg);
        fpu_discard(curlwp, 0);
}