/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 * Copyright 2021 Joyent, Inc.
 * Copyright 2021 Oxide Computer Company
 */

#include <sys/types.h>
#include <sys/systm.h>
#include <sys/schedctl.h>
#include <sys/proc.h>
#include <sys/thread.h>
#include <sys/class.h>
#include <sys/cred.h>
#include <sys/kmem.h>
#include <sys/cmn_err.h>
#include <sys/stack.h>
#include <sys/debug.h>
#include <sys/cpuvar.h>
#include <sys/sobject.h>
#include <sys/door.h>
#include <sys/modctl.h>
#include <sys/syscall.h>
#include <sys/sysmacros.h>
#include <sys/vmsystm.h>
#include <sys/mman.h>
#include <sys/vnode.h>
#include <sys/swap.h>
#include <sys/lwp.h>
#include <sys/bitmap.h>
#include <sys/atomic.h>
#include <sys/fcntl.h>
#include <vm/seg_kp.h>
#include <vm/seg_vn.h>
#include <vm/as.h>
#include <fs/fs_subr.h>

/*
 * Page handling structures.  This is set up as a list of per-page
 * control structures (sc_page_ctl), with p->p_pagep pointing to
 * the first.  The per-page structures point to the actual pages
 * and contain pointers to the user address for each mapped page.
 *
 * All data is protected by p->p_sc_lock.  Since this lock is
 * held while waiting for memory, schedctl_shared_alloc() should
 * not be called while holding p_lock.
 */

typedef struct sc_page_ctl {
        struct sc_page_ctl *spc_next;
        sc_shared_t     *spc_base;      /* base of kernel page */
        sc_shared_t     *spc_end;       /* end of usable space */
        ulong_t         *spc_map;       /* bitmap of allocated space on page */
        size_t          spc_space;      /* amount of space on page */
        caddr_t         spc_uaddr;      /* user-level address of the page */
        struct anon_map *spc_amp;       /* anonymous memory structure */
} sc_page_ctl_t;

static size_t   sc_pagesize;            /* size of usable space on page */
static size_t   sc_bitmap_len;          /* # of bits in allocation bitmap */
static size_t   sc_bitmap_words;        /* # of words in allocation bitmap */

/* Context ops */
static void schedctl_save(void *);
static void schedctl_restore(void *);
static void schedctl_fork(void *, void *);

/* Functions for handling shared pages */
static int      schedctl_shared_alloc(sc_shared_t **, uintptr_t *);
static sc_page_ctl_t *schedctl_page_lookup(sc_shared_t *);
static int      schedctl_map(struct anon_map *, caddr_t *, caddr_t);
static int      schedctl_getpage(struct anon_map **, caddr_t *);
static void     schedctl_freepage(struct anon_map *, caddr_t);

static const struct ctxop_template schedctl_ctxop_tpl = {
        .ct_rev         = CTXOP_TPL_REV,
        .ct_save        = schedctl_save,
        .ct_restore     = schedctl_restore,
        .ct_fork        = schedctl_fork,
};

/*
 * System call interface to scheduler activations.
 * This always operates on the current lwp.
 */
caddr_t
schedctl(void)
{
        kthread_t       *t = curthread;
        sc_shared_t     *ssp;
        uintptr_t       uaddr;
        int             error;

        if (t->t_schedctl == NULL) {
                /*
                 * Allocate and initialize the shared structure.
                 */
                if ((error = schedctl_shared_alloc(&ssp, &uaddr)) != 0)
                        return ((caddr_t)(uintptr_t)set_errno(error));
                bzero(ssp, sizeof (*ssp));

                ctxop_install(t, &schedctl_ctxop_tpl, ssp);

                thread_lock(t); /* protect against ts_tick and ts_update */
                t->t_schedctl = ssp;
                t->t_sc_uaddr = uaddr;
                ssp->sc_cid = t->t_cid;
                ssp->sc_cpri = t->t_cpri;
                ssp->sc_priority = DISP_PRIO(t);
                thread_unlock(t);
        }

        return ((caddr_t)t->t_sc_uaddr);
}


/*
 * Clean up scheduler activations state associated with an exiting
 * (or execing) lwp.  t is always the current thread.
 */
void
schedctl_lwp_cleanup(kthread_t *t)
{
        sc_shared_t     *ssp = t->t_schedctl;
        proc_t          *p = ttoproc(t);
        sc_page_ctl_t   *pagep;
        index_t         index;

        ASSERT(MUTEX_NOT_HELD(&p->p_lock));

        thread_lock(t);         /* protect against ts_tick and ts_update */
        t->t_schedctl = NULL;
        t->t_sc_uaddr = 0;
        thread_unlock(t);

        /*
         * Remove the context op to avoid the final call to
         * schedctl_save when switching away from this lwp.
         */
        (void) ctxop_remove(t, &schedctl_ctxop_tpl, ssp);

        /*
         * Do not unmap the shared page until the process exits.
         * User-level library code relies on this for adaptive mutex locking.
         */
        mutex_enter(&p->p_sc_lock);
        ssp->sc_state = SC_FREE;
        pagep = schedctl_page_lookup(ssp);
        index = (index_t)(ssp - pagep->spc_base);
        BT_CLEAR(pagep->spc_map, index);
        pagep->spc_space += sizeof (sc_shared_t);
        mutex_exit(&p->p_sc_lock);
}


/*
 * Cleanup the list of schedctl shared pages for the process.
 * Called from exec() and exit() system calls.
 */
void
schedctl_proc_cleanup(void)
{
        proc_t *p = curproc;
        sc_page_ctl_t *pagep;
        sc_page_ctl_t *next;

        ASSERT(p->p_lwpcnt == 1);       /* we are single-threaded now */
        ASSERT(curthread->t_schedctl == NULL);

        /*
         * Since we are single-threaded, we don't have to hold p->p_sc_lock.
         */
        pagep = p->p_pagep;
        p->p_pagep = NULL;
        while (pagep != NULL) {
                ASSERT(pagep->spc_space == sc_pagesize);
                next = pagep->spc_next;
                /*
                 * Unmap the user space and free the mapping structure.
                 */
                (void) as_unmap(p->p_as, pagep->spc_uaddr, PAGESIZE);
                schedctl_freepage(pagep->spc_amp, (caddr_t)(pagep->spc_base));
                kmem_free(pagep->spc_map, sizeof (ulong_t) * sc_bitmap_words);
                kmem_free(pagep, sizeof (sc_page_ctl_t));
                pagep = next;
        }
}


/*
 * Called by resume just before switching away from the current thread.
 * Save new thread state.
 */
static void
schedctl_save(void *arg)
{
        sc_shared_t *ssp = arg;

        ssp->sc_state = curthread->t_state;
}


/*
 * Called by resume after switching to the current thread.
 * Save new thread state and CPU.
 */
static void
schedctl_restore(void *arg)
{
        sc_shared_t *ssp = arg;

        ssp->sc_state = SC_ONPROC;
        ssp->sc_cpu = CPU->cpu_id;
}


/*
 * On fork, remove inherited mappings from the child's address space.
 * The child's threads must call schedctl() to get new shared mappings.
 */
static void
schedctl_fork(void *parent, void *child)
{
        kthread_t *pt = parent, *ct = child;
        proc_t *pp = ttoproc(pt);
        proc_t *cp = ttoproc(ct);
        sc_page_ctl_t *pagep;

        ASSERT(ct->t_schedctl == NULL);

        /*
         * Do this only once, whether we are doing fork1() or forkall().
         * Don't do it at all if the child process is a child of vfork()
         * because a child of vfork() borrows the parent's address space.
         */
        if (pt != curthread || (cp->p_flag & SVFORK))
                return;

        mutex_enter(&pp->p_sc_lock);
        for (pagep = pp->p_pagep; pagep != NULL; pagep = pagep->spc_next)
                (void) as_unmap(cp->p_as, pagep->spc_uaddr, PAGESIZE);
        mutex_exit(&pp->p_sc_lock);
}


/*
 * Returns non-zero if the specified thread shouldn't be preempted at this time.
 * Called by ts_preempt(), ts_tick(), and ts_update().
 */
int
schedctl_get_nopreempt(kthread_t *t)
{
        ASSERT(THREAD_LOCK_HELD(t));
        return (t->t_schedctl->sc_preemptctl.sc_nopreempt);
}


/*
 * Sets the value of the nopreempt field for the specified thread.
 * Called by ts_preempt() to clear the field on preemption.
 */
void
schedctl_set_nopreempt(kthread_t *t, short val)
{
        ASSERT(THREAD_LOCK_HELD(t));
        t->t_schedctl->sc_preemptctl.sc_nopreempt = val;
}


/*
 * Sets the value of the yield field for the specified thread.
 * Called by ts_preempt() and ts_tick() to set the field, and
 * ts_yield() to clear it.
 * The kernel never looks at this field so we don't need a
 * schedctl_get_yield() function.
 */
void
schedctl_set_yield(kthread_t *t, short val)
{
        ASSERT(THREAD_LOCK_HELD(t));
        t->t_schedctl->sc_preemptctl.sc_yield = val;
}


/*
 * Sets the values of the cid and priority fields for the specified thread.
 * Called from thread_change_pri(), thread_change_epri(), THREAD_CHANGE_PRI().
 * Called following calls to CL_FORKRET() and CL_ENTERCLASS().
 */
void
schedctl_set_cidpri(kthread_t *t)
{
        sc_shared_t *tdp = t->t_schedctl;

        if (tdp != NULL) {
                tdp->sc_cid = t->t_cid;
                tdp->sc_cpri = t->t_cpri;
                tdp->sc_priority = DISP_PRIO(t);
        }
}


/*
 * Returns non-zero if the specified thread has requested that all
 * signals be blocked.  Called by signal-related code that tests
 * the signal mask of a thread that may not be the current thread
 * and where the process's p_lock cannot be acquired.
 */
int
schedctl_sigblock(kthread_t *t)
{
        sc_shared_t *tdp = t->t_schedctl;

        if (tdp != NULL)
                return (tdp->sc_sigblock);
        return (0);
}


/*
 * If the sc_sigblock field is set for the specified thread, set its signal
 * mask to block all maskable signals, then clear the sc_sigblock field.  This
 * accomplishes what user-level code requested to be done when it set
 * tdp->sc_shared->sc_sigblock non-zero.
 *
 * This is generally called by signal-related code in the current thread.  In
 * order to call against a thread other than curthread, p_lock for the
 * containing process must be held.  Even then, the caller is not protected
 * from races with the thread in question updating its own fields.  It is the
 * responsibility of the caller to perform additional synchronization.
 *
 */
void
schedctl_finish_sigblock(kthread_t *t)
{
        sc_shared_t *tdp = t->t_schedctl;

        ASSERT(t == curthread || MUTEX_HELD(&ttoproc(t)->p_lock));

        if (tdp != NULL && tdp->sc_sigblock) {
                t->t_hold.__sigbits[0] = FILLSET0 & ~CANTMASK0;
                t->t_hold.__sigbits[1] = FILLSET1 & ~CANTMASK1;
                t->t_hold.__sigbits[2] = FILLSET2 & ~CANTMASK2;
                tdp->sc_sigblock = 0;
        }
}


/*
 * Return non-zero if the current thread has declared that it has
 * a cancellation pending and that cancellation is not disabled.
 * If SIGCANCEL is blocked, we must be going over the wire in an
 * NFS transaction (sigintr() was called); return zero in this case.
 */
int
schedctl_cancel_pending(void)
{
        sc_shared_t *tdp = curthread->t_schedctl;

        if (tdp != NULL &&
            (tdp->sc_flgs & SC_CANCEL_FLG) &&
            !tdp->sc_sigblock &&
            !sigismember(&curthread->t_hold, SIGCANCEL))
                return (1);
        return (0);
}


/*
 * Inform libc that the kernel returned EINTR from some system call
 * due to there being a cancellation pending (SC_CANCEL_FLG set or
 * we received an SI_LWP SIGCANCEL while in a system call), rather
 * than because of some other signal.  User-level code can try to
 * recover from receiving other signals, but it can't recover from
 * being cancelled.
 */
void
schedctl_cancel_eintr(void)
{
        sc_shared_t *tdp = curthread->t_schedctl;

        if (tdp != NULL)
                tdp->sc_flgs |= SC_EINTR_FLG;
}


/*
 * Return non-zero if the current thread has declared that
 * it is calling into the kernel to park, else return zero.
 */
int
schedctl_is_park(void)
{
        sc_shared_t *tdp = curthread->t_schedctl;

        if (tdp != NULL)
                return ((tdp->sc_flgs & SC_PARK_FLG) != 0);
        /*
         * If we're here and there is no shared memory (how could
         * that happen?) then just assume we really are here to park.
         */
        return (1);
}


/*
 * Declare thread is parking.
 *
 * libc will set "sc_flgs |= SC_PARK_FLG" before calling lwpsys_park(0, tid)
 * in order to declare that the thread is calling into the kernel to park.
 *
 * This interface exists ONLY to support older versions of libthread which
 * are not aware of the SC_PARK_FLG flag.
 *
 * Older versions of libthread which are not aware of the SC_PARK_FLG flag
 * need to be modified or emulated to call lwpsys_park(4, ...) instead of
 * lwpsys_park(0, ...).  This will invoke schedctl_set_park() before
 * lwp_park() to declare that the thread is parking.
 */
void
schedctl_set_park(void)
{
        sc_shared_t *tdp = curthread->t_schedctl;
        if (tdp != NULL)
                tdp->sc_flgs |= SC_PARK_FLG;
}


/*
 * Clear the parking flag on return from parking in the kernel.
 */
void
schedctl_unpark(void)
{
        sc_shared_t *tdp = curthread->t_schedctl;

        if (tdp != NULL)
                tdp->sc_flgs &= ~SC_PARK_FLG;
}


/*
 * Page handling code.
 */

void
schedctl_init(void)
{
        /*
         * Amount of page that can hold sc_shared_t structures.  If
         * sizeof (sc_shared_t) is a power of 2, this should just be
         * PAGESIZE.
         */
        sc_pagesize = PAGESIZE - (PAGESIZE % sizeof (sc_shared_t));

        /*
         * Allocation bitmap is one bit per struct on a page.
         */
        sc_bitmap_len = sc_pagesize / sizeof (sc_shared_t);
        sc_bitmap_words = howmany(sc_bitmap_len, BT_NBIPUL);
}


static int
schedctl_shared_alloc(sc_shared_t **kaddrp, uintptr_t *uaddrp)
{
        proc_t          *p = curproc;
        sc_page_ctl_t   *pagep;
        sc_shared_t     *ssp;
        caddr_t         base;
        index_t         index;
        int             error;

        ASSERT(MUTEX_NOT_HELD(&p->p_lock));
        mutex_enter(&p->p_sc_lock);

        /*
         * Try to find space for the new data in existing pages
         * within the process's list of shared pages.
         */
        for (pagep = p->p_pagep; pagep != NULL; pagep = pagep->spc_next)
                if (pagep->spc_space != 0)
                        break;

        if (pagep != NULL)
                base = pagep->spc_uaddr;
        else {
                struct anon_map *amp;
                caddr_t kaddr;

                /*
                 * No room, need to allocate a new page.  Also set up
                 * a mapping to the kernel address space for the new
                 * page and lock it in memory.
                 */
                if ((error = schedctl_getpage(&amp, &kaddr)) != 0) {
                        mutex_exit(&p->p_sc_lock);
                        return (error);
                }
                if ((error = schedctl_map(amp, &base, kaddr)) != 0) {
                        schedctl_freepage(amp, kaddr);
                        mutex_exit(&p->p_sc_lock);
                        return (error);
                }

                /*
                 * Allocate and initialize the page control structure.
                 */
                pagep = kmem_alloc(sizeof (sc_page_ctl_t), KM_SLEEP);
                pagep->spc_amp = amp;
                pagep->spc_base = (sc_shared_t *)kaddr;
                pagep->spc_end = (sc_shared_t *)(kaddr + sc_pagesize);
                pagep->spc_uaddr = base;

                pagep->spc_map = kmem_zalloc(sizeof (ulong_t) * sc_bitmap_words,
                    KM_SLEEP);
                pagep->spc_space = sc_pagesize;

                pagep->spc_next = p->p_pagep;
                p->p_pagep = pagep;
        }

        /*
         * Got a page, now allocate space for the data.  There should
         * be space unless something's wrong.
         */
        ASSERT(pagep != NULL && pagep->spc_space >= sizeof (sc_shared_t));
        index = bt_availbit(pagep->spc_map, sc_bitmap_len);
        ASSERT(index != -1);

        /*
         * Get location with pointer arithmetic.  spc_base is of type
         * sc_shared_t *.  Mark as allocated.
         */
        ssp = pagep->spc_base + index;
        BT_SET(pagep->spc_map, index);
        pagep->spc_space -= sizeof (sc_shared_t);

        mutex_exit(&p->p_sc_lock);

        /*
         * Return kernel and user addresses.
         */
        *kaddrp = ssp;
        *uaddrp = (uintptr_t)base + ((uintptr_t)ssp & PAGEOFFSET);
        return (0);
}


/*
 * Find the page control structure corresponding to a kernel address.
 */
static sc_page_ctl_t *
schedctl_page_lookup(sc_shared_t *ssp)
{
        proc_t *p = curproc;
        sc_page_ctl_t *pagep;

        ASSERT(MUTEX_HELD(&p->p_sc_lock));
        for (pagep = p->p_pagep; pagep != NULL; pagep = pagep->spc_next) {
                if (ssp >= pagep->spc_base && ssp < pagep->spc_end)
                        return (pagep);
        }
        return (NULL);          /* This "can't happen".  Should we panic? */
}


/*
 * This function is called when a page needs to be mapped into a
 * process's address space.  Allocate the user address space and
 * set up the mapping to the page.  Assumes the page has already
 * been allocated and locked in memory via schedctl_getpage.
 */
static int
schedctl_map(struct anon_map *amp, caddr_t *uaddrp, caddr_t kaddr)
{
        caddr_t addr = NULL;
        struct as *as = curproc->p_as;
        struct segvn_crargs vn_a;
        int error;

        as_rangelock(as);
        /* pass address of kernel mapping as offset to avoid VAC conflicts */
        map_addr(&addr, PAGESIZE, (offset_t)(uintptr_t)kaddr, 1, 0);
        if (addr == NULL) {
                as_rangeunlock(as);
                return (ENOMEM);
        }

        /*
         * Use segvn to set up the mapping to the page.
         */
        vn_a.vp = NULL;
        vn_a.offset = 0;
        vn_a.cred = NULL;
        vn_a.type = MAP_SHARED;
        vn_a.prot = vn_a.maxprot = PROT_ALL;
        vn_a.flags = 0;
        vn_a.amp = amp;
        vn_a.szc = 0;
        vn_a.lgrp_mem_policy_flags = 0;
        error = as_map(as, addr, PAGESIZE, segvn_create, &vn_a);
        as_rangeunlock(as);

        if (error)
                return (error);

        *uaddrp = addr;
        return (0);
}


/*
 * Allocate a new page from anonymous memory.  Also, create a kernel
 * mapping to the page and lock the page in memory.
 */
static int
schedctl_getpage(struct anon_map **newamp, caddr_t *newaddr)
{
        struct anon_map *amp;
        caddr_t kaddr;

        /*
         * Set up anonymous memory struct.  No swap reservation is
         * needed since the page will be locked into memory.
         */
        amp = anonmap_alloc(PAGESIZE, 0, ANON_SLEEP);

        /*
         * Allocate the page.
         */
        kaddr = segkp_get_withanonmap(segkp, PAGESIZE,
            KPD_NO_ANON | KPD_LOCKED | KPD_ZERO, amp);
        if (kaddr == NULL) {
                amp->refcnt--;
                anonmap_free(amp);
                return (ENOMEM);
        }

        /*
         * The page is left SE_SHARED locked so that it won't be
         * paged out or relocated (KPD_LOCKED above).
         */

        *newamp = amp;
        *newaddr = kaddr;
        return (0);
}


/*
 * Take the necessary steps to allow a page to be released.
 * This is called when the process is doing exit() or exec().
 * There should be no accesses to the page after this.
 * The kernel mapping of the page is released and the page is unlocked.
 */
static void
schedctl_freepage(struct anon_map *amp, caddr_t kaddr)
{
        /*
         * Release the lock on the page and remove the kernel mapping.
         */
        ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
        segkp_release(segkp, kaddr);

        /*
         * Decrement the refcnt so the anon_map structure will be freed.
         */
        if (--amp->refcnt == 0) {
                /*
                 * The current process no longer has the page mapped, so
                 * we have to free everything rather than letting as_free
                 * do the work.
                 */
                anonmap_purge(amp);
                anon_free(amp->ahp, 0, PAGESIZE);
                ANON_LOCK_EXIT(&amp->a_rwlock);
                anonmap_free(amp);
        } else {
                ANON_LOCK_EXIT(&amp->a_rwlock);
        }
}