/*      $NetBSD: subr_ipi.c,v 1.11 2023/02/24 11:02:27 riastradh Exp $  */

/*-
 * Copyright (c) 2014 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Mindaugas Rasiukevicius.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Inter-processor interrupt (IPI) interface: asynchronous IPIs to
 * invoke functions with a constant argument and synchronous IPIs
 * with the cross-call support.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: subr_ipi.c,v 1.11 2023/02/24 11:02:27 riastradh Exp $");

#include <sys/param.h>
#include <sys/types.h>

#include <sys/atomic.h>
#include <sys/evcnt.h>
#include <sys/cpu.h>
#include <sys/ipi.h>
#include <sys/intr.h>
#include <sys/kcpuset.h>
#include <sys/kmem.h>
#include <sys/lock.h>
#include <sys/mutex.h>

/*
 * An array of the IPI handlers used for asynchronous invocation.
 * The lock protects the slot allocation.
 */

typedef struct {
        ipi_func_t      func;
        void *          arg;
} ipi_intr_t;

static kmutex_t         ipi_mngmt_lock;
static ipi_intr_t       ipi_intrs[IPI_MAXREG]   __cacheline_aligned;

/*
 * Per-CPU mailbox for IPI messages: it is a single cache line storing
 * up to IPI_MSG_MAX messages.  This interface is built on top of the
 * synchronous IPIs.
 */

#define IPI_MSG_SLOTS   (CACHE_LINE_SIZE / sizeof(ipi_msg_t *))
#define IPI_MSG_MAX     IPI_MSG_SLOTS

typedef struct {
        ipi_msg_t *     msg[IPI_MSG_SLOTS];
} ipi_mbox_t;


/* Mailboxes for the synchronous IPIs. */
static ipi_mbox_t *     ipi_mboxes      __read_mostly;
static struct evcnt     ipi_mboxfull_ev __cacheline_aligned;
static void             ipi_msg_cpu_handler(void *);

/* Handler for the synchronous IPIs - it must be zero. */
#define IPI_SYNCH_ID    0

#ifndef MULTIPROCESSOR
#define cpu_ipi(ci)     KASSERT(ci == NULL)
#endif

void
ipi_sysinit(void)
{

        mutex_init(&ipi_mngmt_lock, MUTEX_DEFAULT, IPL_NONE);
        memset(ipi_intrs, 0, sizeof(ipi_intrs));

        /*
         * Register the handler for synchronous IPIs.  This mechanism
         * is built on top of the asynchronous interface.  Slot zero is
         * reserved permanently; it is also handy to use zero as a failure
         * for other registers (as it is potentially less error-prone).
         */
        ipi_intrs[IPI_SYNCH_ID].func = ipi_msg_cpu_handler;

        evcnt_attach_dynamic(&ipi_mboxfull_ev, EVCNT_TYPE_MISC, NULL,
           "ipi", "full");
}

void
ipi_percpu_init(void)
{
        const size_t len = ncpu * sizeof(ipi_mbox_t);

        /* Initialise the per-CPU bit fields. */
        for (u_int i = 0; i < ncpu; i++) {
                struct cpu_info *ci = cpu_lookup(i);
                memset(&ci->ci_ipipend, 0, sizeof(ci->ci_ipipend));
        }

        /* Allocate per-CPU IPI mailboxes. */
        ipi_mboxes = kmem_zalloc(len, KM_SLEEP);
        KASSERT(ipi_mboxes != NULL);
}

/*
 * ipi_register: register an asynchronous IPI handler.
 *
 * => Returns IPI ID which is greater than zero; on failure - zero.
 */
u_int
ipi_register(ipi_func_t func, void *arg)
{
        mutex_enter(&ipi_mngmt_lock);
        for (u_int i = 0; i < IPI_MAXREG; i++) {
                if (ipi_intrs[i].func == NULL) {
                        /* Register the function. */
                        ipi_intrs[i].func = func;
                        ipi_intrs[i].arg = arg;
                        mutex_exit(&ipi_mngmt_lock);

                        KASSERT(i != IPI_SYNCH_ID);
                        return i;
                }
        }
        mutex_exit(&ipi_mngmt_lock);
        printf("WARNING: ipi_register: table full, increase IPI_MAXREG\n");
        return 0;
}

/*
 * ipi_unregister: release the IPI handler given the ID.
 */
void
ipi_unregister(u_int ipi_id)
{
        ipi_msg_t ipimsg = { .func = __FPTRCAST(ipi_func_t, nullop) };

        KASSERT(ipi_id != IPI_SYNCH_ID);
        KASSERT(ipi_id < IPI_MAXREG);

        /* Release the slot. */
        mutex_enter(&ipi_mngmt_lock);
        KASSERT(ipi_intrs[ipi_id].func != NULL);
        ipi_intrs[ipi_id].func = NULL;

        /* Ensure that there are no IPIs in flight. */
        kpreempt_disable();
        ipi_broadcast(&ipimsg, false);
        ipi_wait(&ipimsg);
        kpreempt_enable();
        mutex_exit(&ipi_mngmt_lock);
}

/*
 * ipi_mark_pending: internal routine to mark an IPI pending on the
 * specified CPU (which might be curcpu()).
 */
static bool
ipi_mark_pending(u_int ipi_id, struct cpu_info *ci)
{
        const u_int i = ipi_id >> IPI_BITW_SHIFT;
        const uint32_t bitm = 1U << (ipi_id & IPI_BITW_MASK);

        KASSERT(ipi_id < IPI_MAXREG);
        KASSERT(kpreempt_disabled());

        /* Mark as pending and return true if not previously marked. */
        if ((atomic_load_acquire(&ci->ci_ipipend[i]) & bitm) == 0) {
                membar_release();
                atomic_or_32(&ci->ci_ipipend[i], bitm);
                return true;
        }
        return false;
}

/*
 * ipi_trigger: asynchronously send an IPI to the specified CPU.
 */
void
ipi_trigger(u_int ipi_id, struct cpu_info *ci)
{

        KASSERT(curcpu() != ci);
        if (ipi_mark_pending(ipi_id, ci)) {
                cpu_ipi(ci);
        }
}

/*
 * ipi_trigger_multi_internal: the guts of ipi_trigger_multi() and
 * ipi_trigger_broadcast().
 */
static void
ipi_trigger_multi_internal(u_int ipi_id, const kcpuset_t *target,
    bool skip_self)
{
        const cpuid_t selfid = cpu_index(curcpu());
        CPU_INFO_ITERATOR cii;
        struct cpu_info *ci;

        KASSERT(kpreempt_disabled());
        KASSERT(target != NULL);

        for (CPU_INFO_FOREACH(cii, ci)) {
                const cpuid_t cpuid = cpu_index(ci);

                if (!kcpuset_isset(target, cpuid) || cpuid == selfid) {
                        continue;
                }
                ipi_trigger(ipi_id, ci);
        }
        if (!skip_self && kcpuset_isset(target, selfid)) {
                ipi_mark_pending(ipi_id, curcpu());
                int s = splhigh();
                ipi_cpu_handler();
                splx(s);
        }
}

/*
 * ipi_trigger_multi: same as ipi_trigger() but sends to the multiple
 * CPUs given the target CPU set.
 */
void
ipi_trigger_multi(u_int ipi_id, const kcpuset_t *target)
{
        ipi_trigger_multi_internal(ipi_id, target, false);
}

/*
 * ipi_trigger_broadcast: same as ipi_trigger_multi() to kcpuset_attached,
 * optionally skipping the sending CPU.
 */
void
ipi_trigger_broadcast(u_int ipi_id, bool skip_self)
{
        ipi_trigger_multi_internal(ipi_id, kcpuset_attached, skip_self);
}

/*
 * put_msg: insert message into the mailbox.
 *
 * Caller is responsible for issuing membar_release first.
 */
static inline void
put_msg(ipi_mbox_t *mbox, ipi_msg_t *msg)
{
        int count = SPINLOCK_BACKOFF_MIN;
again:
        for (u_int i = 0; i < IPI_MSG_MAX; i++) {
                if (atomic_cas_ptr(&mbox->msg[i], NULL, msg) == NULL) {
                        return;
                }
        }

        /* All slots are full: we have to spin-wait. */
        ipi_mboxfull_ev.ev_count++;
        SPINLOCK_BACKOFF(count);
        goto again;
}

/*
 * ipi_cpu_handler: the IPI handler.
 */
void
ipi_cpu_handler(void)
{
        struct cpu_info * const ci = curcpu();

        /*
         * Handle asynchronous IPIs: inspect per-CPU bit field, extract
         * IPI ID numbers and execute functions in those slots.
         */
        for (u_int i = 0; i < IPI_BITWORDS; i++) {
                uint32_t pending, bit;

                if (atomic_load_relaxed(&ci->ci_ipipend[i]) == 0) {
                        continue;
                }
                pending = atomic_swap_32(&ci->ci_ipipend[i], 0);
                membar_acquire();
                while ((bit = ffs(pending)) != 0) {
                        const u_int ipi_id = (i << IPI_BITW_SHIFT) | --bit;
                        ipi_intr_t *ipi_hdl = &ipi_intrs[ipi_id];

                        pending &= ~(1U << bit);
                        KASSERT(ipi_hdl->func != NULL);
                        ipi_hdl->func(ipi_hdl->arg);
                }
        }
}

/*
 * ipi_msg_cpu_handler: handle synchronous IPIs - iterate mailbox,
 * execute the passed functions and acknowledge the messages.
 */
static void
ipi_msg_cpu_handler(void *arg __unused)
{
        const struct cpu_info * const ci = curcpu();
        ipi_mbox_t *mbox = &ipi_mboxes[cpu_index(ci)];

        for (u_int i = 0; i < IPI_MSG_MAX; i++) {
                ipi_msg_t *msg;

                /* Get the message. */
                if ((msg = atomic_load_acquire(&mbox->msg[i])) == NULL) {
                        continue;
                }
                atomic_store_relaxed(&mbox->msg[i], NULL);

                /* Execute the handler. */
                KASSERT(msg->func);
                msg->func(msg->arg);

                /* Ack the request. */
                membar_release();
                atomic_dec_uint(&msg->_pending);
        }
}

/*
 * ipi_unicast: send an IPI to a single CPU.
 *
 * => The CPU must be remote; must not be local.
 * => The caller must ipi_wait() on the message for completion.
 */
void
ipi_unicast(ipi_msg_t *msg, struct cpu_info *ci)
{
        const cpuid_t id = cpu_index(ci);

        KASSERT(msg->func != NULL);
        KASSERT(kpreempt_disabled());
        KASSERT(curcpu() != ci);

        msg->_pending = 1;
        membar_release();

        put_msg(&ipi_mboxes[id], msg);
        ipi_trigger(IPI_SYNCH_ID, ci);
}

/*
 * ipi_multicast: send an IPI to each CPU in the specified set.
 *
 * => The caller must ipi_wait() on the message for completion.
 */
void
ipi_multicast(ipi_msg_t *msg, const kcpuset_t *target)
{
        const struct cpu_info * const self = curcpu();
        CPU_INFO_ITERATOR cii;
        struct cpu_info *ci;
        u_int local;

        KASSERT(msg->func != NULL);
        KASSERT(kpreempt_disabled());

        local = !!kcpuset_isset(target, cpu_index(self));
        msg->_pending = kcpuset_countset(target) - local;
        membar_release();

        for (CPU_INFO_FOREACH(cii, ci)) {
                cpuid_t id;

                if (__predict_false(ci == self)) {
                        continue;
                }
                id = cpu_index(ci);
                if (!kcpuset_isset(target, id)) {
                        continue;
                }
                put_msg(&ipi_mboxes[id], msg);
                ipi_trigger(IPI_SYNCH_ID, ci);
        }
        if (local) {
                msg->func(msg->arg);
        }
}

/*
 * ipi_broadcast: send an IPI to all CPUs.
 *
 * => The caller must ipi_wait() on the message for completion.
 */
void
ipi_broadcast(ipi_msg_t *msg, bool skip_self)
{
        const struct cpu_info * const self = curcpu();
        CPU_INFO_ITERATOR cii;
        struct cpu_info *ci;

        KASSERT(msg->func != NULL);
        KASSERT(kpreempt_disabled());

        msg->_pending = ncpu - 1;
        membar_release();

        /* Broadcast IPIs for remote CPUs. */
        for (CPU_INFO_FOREACH(cii, ci)) {
                cpuid_t id;

                if (__predict_false(ci == self)) {
                        continue;
                }
                id = cpu_index(ci);
                put_msg(&ipi_mboxes[id], msg);
                ipi_trigger(IPI_SYNCH_ID, ci);
        }

        if (!skip_self) {
                /* Finally, execute locally. */
                msg->func(msg->arg);
        }
}

/*
 * ipi_wait: spin-wait until the message is processed.
 */
void
ipi_wait(ipi_msg_t *msg)
{
        int count = SPINLOCK_BACKOFF_MIN;

        while (atomic_load_acquire(&msg->_pending)) {
                KASSERT(atomic_load_relaxed(&msg->_pending) < ncpu);
                SPINLOCK_BACKOFF(count);
        }
}