/*      $NetBSD: kern_proc.c,v 1.283 2026/05/20 21:05:12 andvar Exp $   */

/*-
 * Copyright (c) 1999, 2006, 2007, 2008, 2020, 2023
 *     The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
 * NASA Ames Research Center, and by Andrew Doran.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Copyright (c) 1982, 1986, 1989, 1991, 1993
 *      The Regents of the University of California.  All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 * 3. Neither the name of the University nor the names of its contributors
 *    may be used to endorse or promote products derived from this software
 *    without specific prior written permission.
 *
 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
 * SUCH DAMAGE.
 *
 *      @(#)kern_proc.c 8.7 (Berkeley) 2/14/95
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: kern_proc.c,v 1.283 2026/05/20 21:05:12 andvar Exp $");

#ifdef _KERNEL_OPT
#include "opt_kstack.h"
#include "opt_maxuprc.h"
#include "opt_dtrace.h"
#include "opt_compat_netbsd32.h"
#include "opt_kaslr.h"
#endif

#if defined(__HAVE_COMPAT_NETBSD32) && !defined(COMPAT_NETBSD32) \
    && !defined(_RUMPKERNEL)
#define COMPAT_NETBSD32
#endif

#include <sys/param.h>
#include <sys/types.h>

#include <sys/acct.h>
#include <sys/atomic.h>
#include <sys/buf.h>
#include <sys/compat_stub.h>
#include <sys/cpu.h>
#include <sys/dtrace_bsd.h>
#include <sys/exec.h>
#include <sys/file.h>
#include <sys/filedesc.h>
#include <sys/futex.h>
#include <sys/ioctl.h>
#include <sys/kauth.h>
#include <sys/kernel.h>
#include <sys/kmem.h>
#include <sys/namei.h>
#include <sys/pool.h>
#include <sys/proc.h>
#include <sys/pserialize.h>
#include <sys/pset.h>
#include <sys/ras.h>
#include <sys/resourcevar.h>
#include <sys/sdt.h>
#include <sys/signalvar.h>
#include <sys/sleepq.h>
#include <sys/syscall_stats.h>
#include <sys/sysctl.h>
#include <sys/systm.h>
#include <sys/tty.h>
#include <sys/uio.h>
#include <sys/wait.h>
#include <ufs/ufs/quota.h>

#include <uvm/uvm_extern.h>

/*
 * Process lists.
 */

struct proclist         allproc         __cacheline_aligned;
struct proclist         zombproc        __cacheline_aligned;

kmutex_t                proc_lock       __cacheline_aligned;
static pserialize_t     proc_psz;

/*
 * pid to lwp/proc lookup is done by indexing the pid_table array.
 * Since pid numbers are only allocated when an empty slot
 * has been found, there is no need to search any lists ever.
 * (an orphaned pgrp will lock the slot, a session will lock
 * the pgrp with the same number.)
 * If the table is too small it is reallocated with twice the
 * previous size and the entries 'unzipped' into the two halves.
 * A linked list of free entries is passed through the pt_lwp
 * field of 'free' items - set odd to be an invalid ptr.  Two
 * additional bits are also used to indicate if the slot is
 * currently occupied by a proc or lwp, and if the PID is
 * hidden from certain kinds of lookups.  We thus require a
 * minimum alignment for proc and lwp structures (LWPs are
 * at least 32-byte aligned).
 */

struct pid_table {
        uintptr_t       pt_slot;
        struct pgrp     *pt_pgrp;
        pid_t           pt_pid;
};

#define PT_F_FREE               ((uintptr_t)__BIT(0))
#define PT_F_LWP                0       /* pseudo-flag */
#define PT_F_PROC               ((uintptr_t)__BIT(1))

#define PT_F_TYPEBITS           (PT_F_FREE|PT_F_PROC)
#define PT_F_ALLBITS            (PT_F_FREE|PT_F_PROC)

#define PT_VALID(s)             (((s) & PT_F_FREE) == 0)
#define PT_RESERVED(s)          ((s) == 0)
#define PT_NEXT(s)              ((u_int)(s) >> 1)
#define PT_SET_FREE(pid)        (((pid) << 1) | PT_F_FREE)
#define PT_SET_LWP(l)           ((uintptr_t)(l))
#define PT_SET_PROC(p)          (((uintptr_t)(p)) | PT_F_PROC)
#define PT_SET_RESERVED         0
#define PT_GET_LWP(s)           ((struct lwp *)((s) & ~PT_F_ALLBITS))
#define PT_GET_PROC(s)          ((struct proc *)((s) & ~PT_F_ALLBITS))
#define PT_GET_TYPE(s)          ((s) & PT_F_TYPEBITS)
#define PT_IS_LWP(s)            (PT_GET_TYPE(s) == PT_F_LWP && (s) != 0)
#define PT_IS_PROC(s)           (PT_GET_TYPE(s) == PT_F_PROC)

#define MIN_PROC_ALIGNMENT      (PT_F_ALLBITS + 1)

/*
 * Table of process IDs (PIDs).
 */
static struct pid_table *pid_table      __read_mostly;

#define INITIAL_PID_TABLE_SIZE          (1 << 5)

/* Table mask, threshold for growing and number of allocated PIDs. */
static u_int            pid_tbl_mask    __read_mostly;
static u_int            pid_alloc_lim   __read_mostly;
static u_int            pid_alloc_cnt   __cacheline_aligned;

/* Next free, last free and maximum PIDs. */
static u_int            next_free_pt    __cacheline_aligned;
static u_int            last_free_pt    __cacheline_aligned;
static pid_t            pid_max         __read_mostly;

/* Components of the first process -- never freed. */

struct session session0 = {
        .s_count = 1,
        .s_sid = 0,
};
struct pgrp pgrp0 = {
        .pg_members = LIST_HEAD_INITIALIZER(&pgrp0.pg_members),
        .pg_session = &session0,
};
filedesc_t filedesc0;
struct cwdinfo cwdi0 = {
        .cwdi_cmask = CMASK,
        .cwdi_refcnt = 1,
};
struct plimit limit0;
struct pstats pstat0;
struct vmspace vmspace0;
struct sigacts sigacts0;
struct proc proc0 = {
        .p_lwps = LIST_HEAD_INITIALIZER(&proc0.p_lwps),
        .p_sigwaiters = LIST_HEAD_INITIALIZER(&proc0.p_sigwaiters),
        .p_nlwps = 1,
        .p_nrlwps = 1,
        .p_pgrp = &pgrp0,
        .p_comm = "system",
        /*
         * Set P_NOCLDWAIT so that kernel threads are reparented to init(8)
         * when they exit.  init(8) can easily wait them out for us.
         */
        .p_flag = PK_SYSTEM | PK_NOCLDWAIT,
        .p_stat = SACTIVE,
        .p_nice = NZERO,
        .p_emul = &emul_netbsd,
        .p_cwdi = &cwdi0,
        .p_limit = &limit0,
        .p_fd = &filedesc0,
        .p_vmspace = &vmspace0,
        .p_stats = &pstat0,
        .p_sigacts = &sigacts0,
#ifdef PROC0_MD_INITIALIZERS
        PROC0_MD_INITIALIZERS
#endif
};
kauth_cred_t cred0;

static const int        nofile  = NOFILE;
static const int        maxuprc = MAXUPRC;

static int sysctl_doeproc(SYSCTLFN_PROTO);
static int sysctl_kern_proc_args(SYSCTLFN_PROTO);
static int sysctl_security_expose_address(SYSCTLFN_PROTO);

#ifdef KASLR
static int kern_expose_address = 0;
#else
static int kern_expose_address = 1;
#endif
/*
 * The process list descriptors, used during pid allocation and
 * by sysctl.  No locking on this data structure is needed since
 * it is completely static.
 */
const struct proclist_desc proclists[] = {
        { &allproc      },
        { &zombproc     },
        { NULL          },
};

static struct pgrp *    pg_remove(pid_t);
static void             pg_delete(pid_t);
static void             orphanpg(struct pgrp *);

static specificdata_domain_t proc_specificdata_domain;

static pool_cache_t proc_cache;

static kauth_listener_t proc_listener;

static void fill_proc(const struct proc *, struct proc *, bool);
static int fill_pathname(struct lwp *, pid_t, void *, size_t *);
static int fill_cwd(struct lwp *, pid_t, void *, size_t *);

static int
proc_listener_cb(kauth_cred_t cred, kauth_action_t action, void *cookie,
    void *arg0, void *arg1, void *arg2, void *arg3)
{
        struct proc *p;
        int result;

        result = KAUTH_RESULT_DEFER;
        p = arg0;

        switch (action) {
        case KAUTH_PROCESS_CANSEE: {
                enum kauth_process_req req;

                req = (enum kauth_process_req)(uintptr_t)arg1;

                switch (req) {
                case KAUTH_REQ_PROCESS_CANSEE_ARGS:
                case KAUTH_REQ_PROCESS_CANSEE_ENTRY:
                case KAUTH_REQ_PROCESS_CANSEE_OPENFILES:
                case KAUTH_REQ_PROCESS_CANSEE_EPROC:
                        result = KAUTH_RESULT_ALLOW;
                        break;

                case KAUTH_REQ_PROCESS_CANSEE_ENV:
                        if (kauth_cred_getuid(cred) !=
                            kauth_cred_getuid(p->p_cred) ||
                            kauth_cred_getuid(cred) !=
                            kauth_cred_getsvuid(p->p_cred))
                                break;

                        result = KAUTH_RESULT_ALLOW;

                        break;

                case KAUTH_REQ_PROCESS_CANSEE_KPTR:
                        if (!kern_expose_address)
                                break;

                        if (kern_expose_address == 1 && !(p->p_flag & PK_KMEM))
                                break;

                        result = KAUTH_RESULT_ALLOW;

                        break;

                default:
                        break;
                }

                break;
                }

        case KAUTH_PROCESS_FORK: {
                int lnprocs = (int)(unsigned long)arg2;

                /*
                 * Don't allow a nonprivileged user to use the last few
                 * processes. The variable lnprocs is the current number of
                 * processes, maxproc is the limit.
                 */
                if (__predict_false((lnprocs >= maxproc - 5)))
                        break;

                result = KAUTH_RESULT_ALLOW;

                break;
                }

        case KAUTH_PROCESS_CORENAME:
        case KAUTH_PROCESS_STOPFLAG:
                if (proc_uidmatch(cred, p->p_cred) == 0)
                        result = KAUTH_RESULT_ALLOW;

                break;

        default:
                break;
        }

        return result;
}

static int
proc_ctor(void *arg __unused, void *obj, int flags __unused)
{
        struct proc *p = obj;

        memset(p, 0, sizeof(*p));
        klist_init(&p->p_klist);

        /*
         * There is no need for a proc_dtor() to do a klist_fini(),
         * since knote_proc_exit() ensures that p->p_klist is empty
         * when a process exits.
         */

        return 0;
}

static pid_t proc_alloc_pid_slot(struct proc *, uintptr_t);

/*
 * Initialize global process hashing structures.
 */
void
procinit(void)
{
        const struct proclist_desc *pd;
        u_int i;
#define LINK_EMPTY ((PID_MAX + INITIAL_PID_TABLE_SIZE) & ~(INITIAL_PID_TABLE_SIZE - 1))

        for (pd = proclists; pd->pd_list != NULL; pd++)
                LIST_INIT(pd->pd_list);

        mutex_init(&proc_lock, MUTEX_DEFAULT, IPL_NONE);

        proc_psz = pserialize_create();

        pid_table = kmem_alloc(INITIAL_PID_TABLE_SIZE
            * sizeof(struct pid_table), KM_SLEEP);
        pid_tbl_mask = INITIAL_PID_TABLE_SIZE - 1;
        pid_max = PID_MAX;

        /* Set free list running through table...
           Preset 'use count' above PID_MAX so we allocate pid 1 next. */
        for (i = 0; i <= pid_tbl_mask; i++) {
                pid_table[i].pt_slot = PT_SET_FREE(LINK_EMPTY + i + 1);
                pid_table[i].pt_pgrp = 0;
                pid_table[i].pt_pid = 0;
        }
        /* slot 0 is just grabbed */
        next_free_pt = 1;
        /* Need to fix last entry. */
        last_free_pt = pid_tbl_mask;
        pid_table[last_free_pt].pt_slot = PT_SET_FREE(LINK_EMPTY);
        /* point at which we grow table - to avoid reusing pids too often */
        pid_alloc_lim = pid_tbl_mask - 1;
#undef LINK_EMPTY

        /* Reserve PID 1 for init(8). */        /* XXX slightly gross */
        mutex_enter(&proc_lock);
        if (proc_alloc_pid_slot(&proc0, PT_SET_RESERVED) != 1)
                panic("failed to reserve PID 1 for init(8)");
        mutex_exit(&proc_lock);

        proc_specificdata_domain = specificdata_domain_create();
        KASSERT(proc_specificdata_domain != NULL);

        size_t proc_alignment = coherency_unit;
        if (proc_alignment < MIN_PROC_ALIGNMENT)
                proc_alignment = MIN_PROC_ALIGNMENT;

        proc_cache = pool_cache_init(sizeof(struct proc), proc_alignment, 0, 0,
            "procpl", NULL, IPL_NONE, proc_ctor, NULL, NULL);

        proc_listener = kauth_listen_scope(KAUTH_SCOPE_PROCESS,
            proc_listener_cb, NULL);
}

void
procinit_sysctl(void)
{
        static struct sysctllog *clog;

        sysctl_createv(&clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT|CTLFLAG_READWRITE,
                       CTLTYPE_INT, "expose_address",
                       SYSCTL_DESCR("Enable exposing kernel addresses"),
                       sysctl_security_expose_address, 0,
                       &kern_expose_address, 0, CTL_KERN, CTL_CREATE, CTL_EOL);
        sysctl_createv(&clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT,
                       CTLTYPE_NODE, "proc",
                       SYSCTL_DESCR("System-wide process information"),
                       sysctl_doeproc, 0, NULL, 0,
                       CTL_KERN, KERN_PROC, CTL_EOL);
        sysctl_createv(&clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT,
                       CTLTYPE_NODE, "proc2",
                       SYSCTL_DESCR("Machine-independent process information"),
                       sysctl_doeproc, 0, NULL, 0,
                       CTL_KERN, KERN_PROC2, CTL_EOL);
        sysctl_createv(&clog, 0, NULL, NULL,
                       CTLFLAG_PERMANENT,
                       CTLTYPE_NODE, "proc_args",
                       SYSCTL_DESCR("Process argument information"),
                       sysctl_kern_proc_args, 0, NULL, 0,
                       CTL_KERN, KERN_PROC_ARGS, CTL_EOL);

        /*
          "nodes" under these:

          KERN_PROC_ALL
          KERN_PROC_PID pid
          KERN_PROC_PGRP pgrp
          KERN_PROC_SESSION sess
          KERN_PROC_TTY tty
          KERN_PROC_UID uid
          KERN_PROC_RUID uid
          KERN_PROC_GID gid
          KERN_PROC_RGID gid

          all in all, probably not worth the effort...
        */
}

/*
 * Initialize process 0.
 */
void
proc0_init(void)
{
        struct proc *p;
        struct pgrp *pg;
        struct rlimit *rlim;
        rlim_t lim;
        int i;

        p = &proc0;
        pg = &pgrp0;

        mutex_init(&p->p_stmutex, MUTEX_DEFAULT, IPL_HIGH);
        mutex_init(&p->p_auxlock, MUTEX_DEFAULT, IPL_NONE);
        p->p_lock = mutex_obj_alloc(MUTEX_DEFAULT, IPL_NONE);

        rw_init(&p->p_reflock);
        cv_init(&p->p_waitcv, "wait");
        cv_init(&p->p_lwpcv, "lwpwait");

        LIST_INSERT_HEAD(&p->p_lwps, &lwp0, l_sibling);

        KASSERT(lwp0.l_lid == 0);
        pid_table[lwp0.l_lid].pt_slot = PT_SET_LWP(&lwp0);
        LIST_INSERT_HEAD(&allproc, p, p_list);

        pid_table[lwp0.l_lid].pt_pgrp = pg;
        LIST_INSERT_HEAD(&pg->pg_members, p, p_pglist);

#ifdef __HAVE_SYSCALL_INTERN
        (*p->p_emul->e_syscall_intern)(p);
#endif

        /* Create credentials. */
        cred0 = kauth_cred_alloc();
        p->p_cred = cred0;

        /* Create the CWD info. */
        rw_init(&cwdi0.cwdi_lock);

        /* Create the limits structures. */
        mutex_init(&limit0.pl_lock, MUTEX_DEFAULT, IPL_NONE);

        rlim = limit0.pl_rlimit;
        for (i = 0; i < __arraycount(limit0.pl_rlimit); i++) {
                rlim[i].rlim_cur = RLIM_INFINITY;
                rlim[i].rlim_max = RLIM_INFINITY;
        }

        rlim[RLIMIT_NOFILE].rlim_max = maxfiles;
        rlim[RLIMIT_NOFILE].rlim_cur = maxfiles < nofile ? maxfiles : nofile;

        rlim[RLIMIT_NPROC].rlim_max = maxproc;
        rlim[RLIMIT_NPROC].rlim_cur = maxproc < maxuprc ? maxproc : maxuprc;

        lim = MIN(VM_MAXUSER_ADDRESS, ctob((rlim_t)uvm_availmem(false)));
        rlim[RLIMIT_RSS].rlim_max = lim;
        rlim[RLIMIT_MEMLOCK].rlim_max = lim;
        rlim[RLIMIT_MEMLOCK].rlim_cur = lim / 3;

        rlim[RLIMIT_NTHR].rlim_max = maxlwp;
        rlim[RLIMIT_NTHR].rlim_cur = maxlwp / 2;

        /* Note that default core name has zero length. */
        limit0.pl_corename = defcorename;
        limit0.pl_cnlen = 0;
        limit0.pl_refcnt = 1;
        limit0.pl_writeable = false;
        limit0.pl_sv_limit = NULL;

        /* Configure virtual memory system, set vm rlimits. */
        uvm_init_limits(p);

        /* Initialize file descriptor table for proc0. */
        fd_init(&filedesc0);

        /*
         * Initialize proc0's vmspace, which uses the kernel pmap.
         * All kernel processes (which never have user space mappings)
         * share proc0's vmspace, and thus, the kernel pmap.
         */
        uvmspace_init(&vmspace0, pmap_kernel(), round_page(VM_MIN_ADDRESS),
            trunc_page(VM_MAXUSER_ADDRESS),
#ifdef __USE_TOPDOWN_VM
            true
#else
            false
#endif
            );

        /* Initialize signal state for proc0. XXX IPL_SCHED */
        mutex_init(&p->p_sigacts->sa_mutex, MUTEX_DEFAULT, IPL_SCHED);
        siginit(p);

        proc_initspecific(p);
        kdtrace_proc_ctor(NULL, p);
}

/*
 * Session reference counting.
 */

void
proc_sesshold(struct session *ss)
{

        KASSERT(mutex_owned(&proc_lock));
        ss->s_count++;
}

void
proc_sessrele(struct session *ss)
{
        struct pgrp *pg;

        KASSERT(mutex_owned(&proc_lock));
        KASSERT(ss->s_count > 0);

        /*
         * We keep the pgrp with the same id as the session in order to
         * stop a process being given the same pid.  Since the pgrp holds
         * a reference to the session, it must be a 'zombie' pgrp by now.
         */
        if (--ss->s_count == 0) {
                pg = pg_remove(ss->s_sid);
        } else {
                pg = NULL;
                ss = NULL;
        }

        mutex_exit(&proc_lock);

        if (pg)
                kmem_free(pg, sizeof(struct pgrp));
        if (ss)
                kmem_free(ss, sizeof(struct session));
}

/*
 * Check that the specified process group is in the session of the
 * specified process.
 * Treats -ve ids as process ids.
 * Used to validate TIOCSPGRP requests.
 */
int
pgid_in_session(struct proc *p, pid_t pg_id)
{
        struct pgrp *pgrp;
        struct session *session;
        int error;

        if (pg_id <= INT_MIN)
                return SET_ERROR(EINVAL);

        mutex_enter(&proc_lock);
        if (pg_id < 0) {
                struct proc *p1 = proc_find(-pg_id);
                if (p1 == NULL) {
                        error = SET_ERROR(EINVAL);
                        goto fail;
                }
                pgrp = p1->p_pgrp;
        } else {
                pgrp = pgrp_find(pg_id);
                if (pgrp == NULL) {
                        error = SET_ERROR(EINVAL);
                        goto fail;
                }
        }
        session = pgrp->pg_session;
        error = (session != p->p_pgrp->pg_session) ? SET_ERROR(EPERM) : 0;
fail:
        mutex_exit(&proc_lock);
        return error;
}

/*
 * p_inferior: is p an inferior of q?
 */
static inline bool
p_inferior(struct proc *p, struct proc *q)
{

        KASSERT(mutex_owned(&proc_lock));

        for (; p != q; p = p->p_pptr)
                if (p->p_pid == 0)
                        return false;
        return true;
}

/*
 * proc_find_lwp: locate an lwp in said proc by the ID.
 *
 * => Must be called with p::p_lock held.
 * => LSIDL lwps are not returned because they are only partially
 *    constructed while occupying the slot.
 * => Callers need to be careful about lwp::l_stat of the returned
 *    lwp.
 */
struct lwp *
proc_find_lwp(proc_t *p, pid_t pid)
{
        struct pid_table *pt;
        unsigned pt_mask;
        struct lwp *l = NULL;
        uintptr_t slot;
        int s;

        KASSERT(mutex_owned(p->p_lock));

        /*
         * Look in the pid_table.  This is done unlocked inside a
         * pserialize read section covering pid_table's memory
         * allocation only, so take care to read things in the correct
         * order:
         *
         * 1. First read the table mask -- this only ever increases, in
         *    expand_pid_table, so a stale value is safely
         *    conservative.
         *
         * 2. Next read the pid table -- this is always set _before_
         *    the mask increases, so if we see a new table and stale
         *    mask, the mask is still valid for the table.
         */
        s = pserialize_read_enter();
        pt_mask = atomic_load_acquire(&pid_tbl_mask);
        pt = &atomic_load_consume(&pid_table)[pid & pt_mask];
        slot = atomic_load_consume(&pt->pt_slot);
        if (__predict_false(!PT_IS_LWP(slot))) {
                pserialize_read_exit(s);
                return NULL;
        }

        /*
         * Check to see if the LWP is from the correct process.  We won't
         * see entries in pid_table from a prior process that also used "p",
         * by virtue of the fact that allocating "p" means all prior updates
         * to dependant data structures are visible to this thread.
         */
        l = PT_GET_LWP(slot);
        if (__predict_false(atomic_load_relaxed(&l->l_proc) != p)) {
                pserialize_read_exit(s);
                return NULL;
        }

        /*
         * We now know that p->p_lock holds this LWP stable.
         *
         * If the status is not LSIDL, it means the LWP is intended to be
         * findable by LID and l_lid cannot change behind us.
         *
         * No need to acquire the LWP's lock to check for LSIDL, as
         * p->p_lock must be held to transition in and out of LSIDL.
         * Any other observed state of is no particular interest.
         */
        pserialize_read_exit(s);
        return l->l_stat != LSIDL && l->l_lid == pid ? l : NULL;
}

/*
 * proc_find_lwp_unlocked: locate an lwp in said proc by the ID.
 *
 * => Called in a pserialize read section with no locks held.
 * => LSIDL lwps are not returned because they are only partially
 *    constructed while occupying the slot.
 * => Callers need to be careful about lwp::l_stat of the returned
 *    lwp.
 * => If an LWP is found, it's returned locked.
 */
struct lwp *
proc_find_lwp_unlocked(proc_t *p, pid_t pid)
{
        struct pid_table *pt;
        unsigned pt_mask;
        struct lwp *l = NULL;
        uintptr_t slot;

        KASSERT(pserialize_in_read_section());

        /*
         * Look in the pid_table.  This is done unlocked inside a
         * pserialize read section covering pid_table's memory
         * allocation only, so take care to read things in the correct
         * order:
         *
         * 1. First read the table mask -- this only ever increases, in
         *    expand_pid_table, so a stale value is safely
         *    conservative.
         *
         * 2. Next read the pid table -- this is always set _before_
         *    the mask increases, so if we see a new table and stale
         *    mask, the mask is still valid for the table.
         */
        pt_mask = atomic_load_acquire(&pid_tbl_mask);
        pt = &atomic_load_consume(&pid_table)[pid & pt_mask];
        slot = atomic_load_consume(&pt->pt_slot);
        if (__predict_false(!PT_IS_LWP(slot))) {
                return NULL;
        }

        /*
         * Lock the LWP we found to get it stable.  If it's embryonic or
         * reaped (LSIDL) then none of the other fields can safely be
         * checked.
         */
        l = PT_GET_LWP(slot);
        lwp_lock(l);
        if (__predict_false(l->l_stat == LSIDL)) {
                lwp_unlock(l);
                return NULL;
        }

        /*
         * l_proc and l_lid are now known stable because the LWP is not
         * LSIDL, so check those fields too to make sure we found the
         * right thing.
         */
        if (__predict_false(l->l_proc != p || l->l_lid != pid)) {
                lwp_unlock(l);
                return NULL;
        }

        /* Everything checks out, return it locked. */
        return l;
}

/*
 * proc_find_lwp_acquire_proc: locate an lwp and acquire a lock
 * on its containing proc.
 *
 * => Similar to proc_find_lwp(), but does not require you to have
 *    the proc a priori.
 * => Also returns proc * to caller, with p::p_lock held.
 * => Same caveats apply.
 */
struct lwp *
proc_find_lwp_acquire_proc(pid_t pid, struct proc **pp)
{
        struct pid_table *pt;
        struct proc *p = NULL;
        struct lwp *l = NULL;
        uintptr_t slot;

        KASSERT(pp != NULL);
        mutex_enter(&proc_lock);
        pt = &pid_table[pid & pid_tbl_mask];

        slot = pt->pt_slot;
        if (__predict_true(PT_IS_LWP(slot) && pt->pt_pid == pid)) {
                l = PT_GET_LWP(slot);
                p = l->l_proc;
                mutex_enter(p->p_lock);
                if (__predict_false(l->l_stat == LSIDL)) {
                        mutex_exit(p->p_lock);
                        l = NULL;
                        p = NULL;
                }
        }
        mutex_exit(&proc_lock);

        KASSERT(p == NULL || mutex_owned(p->p_lock));
        *pp = p;
        return l;
}

/*
 * proc_find_raw_pid_table_locked: locate a process by the ID.
 *
 * => Must be called with proc_lock held.
 */
static proc_t *
proc_find_raw_pid_table_locked(pid_t pid, bool any_lwpid)
{
        struct pid_table *pt;
        proc_t *p = NULL;
        uintptr_t slot;

        /* No - used by DDB.  KASSERT(mutex_owned(&proc_lock)); */
        pt = &pid_table[pid & pid_tbl_mask];

        slot = pt->pt_slot;
        if (__predict_true(PT_IS_LWP(slot) && pt->pt_pid == pid)) {
                /*
                 * When looking up processes, require a direct match
                 * on the PID assigned to the proc, not just one of
                 * its LWPs.
                 *
                 * N.B. We require lwp::l_proc of LSIDL LWPs to be
                 * valid here.
                 */
                p = PT_GET_LWP(slot)->l_proc;
                if (__predict_false(p->p_pid != pid && !any_lwpid))
                        p = NULL;
        } else if (PT_IS_PROC(slot) && pt->pt_pid == pid) {
                p = PT_GET_PROC(slot);
        }
        return p;
}

proc_t *
proc_find_raw(pid_t pid)
{

        return proc_find_raw_pid_table_locked(pid, false);
}

static proc_t *
proc_find_internal(pid_t pid, bool any_lwpid)
{
        proc_t *p;

        KASSERT(mutex_owned(&proc_lock));

        p = proc_find_raw_pid_table_locked(pid, any_lwpid);
        if (__predict_false(p == NULL)) {
                return NULL;
        }

        /*
         * Only allow live processes to be found by PID.
         * XXX: p_stat might change, since proc unlocked.
         */
        if (__predict_true(p->p_stat == SACTIVE || p->p_stat == SSTOP)) {
                return p;
        }
        return NULL;
}

proc_t *
proc_find(pid_t pid)
{
        return proc_find_internal(pid, false);
}

proc_t *
proc_find_lwpid(pid_t pid)
{
        return proc_find_internal(pid, true);
}

/*
 * pgrp_find: locate a process group by the ID.
 *
 * => Must be called with proc_lock held.
 */
struct pgrp *
pgrp_find(pid_t pgid)
{
        struct pgrp *pg;

        KASSERT(mutex_owned(&proc_lock));

        pg = pid_table[pgid & pid_tbl_mask].pt_pgrp;

        /*
         * Cannot look up a process group that only exists because the
         * session has not died yet (traditional).
         */
        if (pg == NULL || pg->pg_id != pgid || LIST_EMPTY(&pg->pg_members)) {
                return NULL;
        }
        return pg;
}

static void
expand_pid_table(void)
{
        size_t pt_size, tsz;
        struct pid_table *n_pt, *new_pt;
        uintptr_t slot;
        struct pgrp *pgrp;
        pid_t pid, rpid;
        u_int i;
        uint new_pt_mask;

        KASSERT(mutex_owned(&proc_lock));

        /* Unlock the pid_table briefly to allocate memory. */
        pt_size = pid_tbl_mask + 1;
        mutex_exit(&proc_lock);

        tsz = pt_size * 2 * sizeof(struct pid_table);
        new_pt = kmem_alloc(tsz, KM_SLEEP);
        new_pt_mask = pt_size * 2 - 1;

        /* XXX For now.  The practical limit is much lower anyway. */
        KASSERT(new_pt_mask <= FUTEX_TID_MASK);

        mutex_enter(&proc_lock);
        if (pt_size != pid_tbl_mask + 1) {
                /* Another process beat us to it... */
                mutex_exit(&proc_lock);
                kmem_free(new_pt, tsz);
                goto out;
        }

        /*
         * Copy entries from old table into new one.
         * If 'pid' is 'odd' we need to place in the upper half,
         * even pid's to the lower half.
         * Free items stay in the low half so we don't have to
         * fixup the reference to them.
         * We stuff free items on the front of the freelist
         * because we can't write to unmodified entries.
         * Processing the table backwards maintains a semblance
         * of issuing pid numbers that increase with time.
         */
        i = pt_size - 1;
        n_pt = new_pt + i;
        for (; ; i--, n_pt--) {
                slot = pid_table[i].pt_slot;
                pgrp = pid_table[i].pt_pgrp;
                if (!PT_VALID(slot)) {
                        /* Up 'use count' so that link is valid */
                        pid = (PT_NEXT(slot) + pt_size) & ~pt_size;
                        rpid = 0;
                        slot = PT_SET_FREE(pid);
                        if (pgrp)
                                pid = pgrp->pg_id;
                } else {
                        pid = pid_table[i].pt_pid;
                        rpid = pid;
                }

                /* Save entry in appropriate half of table */
                n_pt[pid & pt_size].pt_slot = slot;
                n_pt[pid & pt_size].pt_pgrp = pgrp;
                n_pt[pid & pt_size].pt_pid = rpid;

                /* Put other piece on start of free list */
                pid = (pid ^ pt_size) & ~pid_tbl_mask;
                n_pt[pid & pt_size].pt_slot =
                        PT_SET_FREE((pid & ~pt_size) | next_free_pt);
                n_pt[pid & pt_size].pt_pgrp = 0;
                n_pt[pid & pt_size].pt_pid = 0;

                next_free_pt = i | (pid & pt_size);
                if (i == 0)
                        break;
        }

        /* Save old table size and switch tables */
        tsz = pt_size * sizeof(struct pid_table);
        n_pt = pid_table;
        atomic_store_release(&pid_table, new_pt);
        KASSERT(new_pt_mask >= pid_tbl_mask);
        atomic_store_release(&pid_tbl_mask, new_pt_mask);

        /*
         * pid_max starts as PID_MAX (= 30000), once we have 16384
         * allocated pids we need it to be larger!
         */
        if (pid_tbl_mask > PID_MAX) {
                pid_max = pid_tbl_mask * 2 + 1;
                pid_alloc_lim |= pid_alloc_lim << 1;
        } else
                pid_alloc_lim <<= 1;    /* doubles number of free slots... */

        mutex_exit(&proc_lock);

        /*
         * Make sure that unlocked access to the old pid_table is complete
         * and then free it.
         */
        pserialize_perform(proc_psz);
        kmem_free(n_pt, tsz);

 out:   /* Return with proc_lock held again. */
        mutex_enter(&proc_lock);
}

struct proc *
proc_alloc(void)
{
        struct proc *p;

        p = pool_cache_get(proc_cache, PR_WAITOK);
        p->p_stat = SIDL;                       /* protect against others */
        proc_initspecific(p);
        kdtrace_proc_ctor(NULL, p);

        /*
         * Allocate a placeholder in the pid_table.  When we create the
         * first LWP for this process, it will take ownership of the
         * slot.
         */
        if (__predict_false(proc_alloc_pid(p) == -1)) {
                /* Allocating the PID failed; unwind. */
                proc_finispecific(p);
                proc_free_mem(p);
                p = NULL;
        }
        return p;
}

/*
 * proc_alloc_pid_slot: allocate PID and record the occupant so that
 * proc_find_raw() can find it by the PID.
 */
static pid_t __noinline
proc_alloc_pid_slot(struct proc *p, uintptr_t slot)
{
        struct pid_table *pt;
        pid_t pid;
        int nxt;

        KASSERT(mutex_owned(&proc_lock));

        for (;;expand_pid_table()) {
                if (__predict_false(pid_alloc_cnt >= pid_alloc_lim)) {
                        /* ensure pids cycle through 2000+ values */
                        continue;
                }
                /*
                 * The first user process *must* be given PID 1.
                 * it has already been reserved for us.  This
                 * will be coming in from the proc_alloc() call
                 * above, and the entry will be usurped later when
                 * the first user LWP is created.
                 * XXX this is slightly gross.
                 */
                if (__predict_false(PT_RESERVED(pid_table[1].pt_slot) &&
                                    p != &proc0)) {
                        KASSERT(PT_IS_PROC(slot));
                        pt = &pid_table[1];
                        pt->pt_slot = slot;
                        return 1;
                }
                pt = &pid_table[next_free_pt];
#ifdef DIAGNOSTIC
                if (__predict_false(PT_VALID(pt->pt_slot) || pt->pt_pgrp))
                        panic("proc_alloc: slot busy");
#endif
                nxt = PT_NEXT(pt->pt_slot);
                if (nxt & pid_tbl_mask)
                        break;
                /* Table full - expand (NB last entry not used....) */
        }

        /* pid is 'saved use count' + 'size' + entry */
        pid = (nxt & ~pid_tbl_mask) + pid_tbl_mask + 1 + next_free_pt;
        if ((uint)pid > (uint)pid_max)
                pid &= pid_tbl_mask;
        next_free_pt = nxt & pid_tbl_mask;

        /* XXX For now.  The practical limit is much lower anyway. */
        KASSERT(pid <= FUTEX_TID_MASK);

        /* Grab table slot */
        pt->pt_slot = slot;

        KASSERT(pt->pt_pid == 0);
        pt->pt_pid = pid;
        pid_alloc_cnt++;

        return pid;
}

pid_t
proc_alloc_pid(struct proc *p)
{
        pid_t pid;

        KASSERT((((uintptr_t)p) & PT_F_ALLBITS) == 0);
        KASSERT(p->p_stat == SIDL);

        mutex_enter(&proc_lock);
        pid = proc_alloc_pid_slot(p, PT_SET_PROC(p));
        if (pid != -1)
                p->p_pid = pid;
        mutex_exit(&proc_lock);

        return pid;
}

pid_t
proc_alloc_lwpid(struct proc *p, struct lwp *l)
{
        struct pid_table *pt;
        pid_t pid;

        KASSERT((((uintptr_t)l) & PT_F_ALLBITS) == 0);
        KASSERT(l->l_proc == p);
        KASSERT(l->l_stat == LSIDL);

        /*
         * For unlocked lookup in proc_find_lwp(), make sure l->l_proc
         * is globally visible before the LWP becomes visible via the
         * pid_table.
         */
#ifndef __HAVE_ATOMIC_AS_MEMBAR
        membar_producer();
#endif

        /*
         * If the slot for p->p_pid currently points to the proc,
         * then we should usurp this ID for the LWP.  This happens
         * at least once per process (for the first LWP), and can
         * happen again if the first LWP for a process exits and
         * before the process creates another.
         */
        mutex_enter(&proc_lock);
        pid = p->p_pid;
        pt = &pid_table[pid & pid_tbl_mask];
        KASSERT(pt->pt_pid == pid);
        if (PT_IS_PROC(pt->pt_slot)) {
                KASSERT(PT_GET_PROC(pt->pt_slot) == p);
                l->l_lid = pid;
                pt->pt_slot = PT_SET_LWP(l);
        } else {
                /* Need to allocate a new slot. */
                pid = proc_alloc_pid_slot(p, PT_SET_LWP(l));
                if (pid != -1)
                        l->l_lid = pid;
        }
        mutex_exit(&proc_lock);

        return pid;
}

static void __noinline
proc_free_pid_internal(pid_t pid, uintptr_t type __diagused)
{
        struct pid_table *pt;

        KASSERT(mutex_owned(&proc_lock));

        pt = &pid_table[pid & pid_tbl_mask];

        KASSERT(PT_GET_TYPE(pt->pt_slot) == type);
        KASSERT(pt->pt_pid == pid);

        /* save pid use count in slot */
        pt->pt_slot = PT_SET_FREE(pid & ~pid_tbl_mask);
        pt->pt_pid = 0;

        if (pt->pt_pgrp == NULL) {
                /* link last freed entry onto ours */
                pid &= pid_tbl_mask;
                pt = &pid_table[last_free_pt];
                pt->pt_slot = PT_SET_FREE(PT_NEXT(pt->pt_slot) | pid);
                pt->pt_pid = 0;
                last_free_pt = pid;
                pid_alloc_cnt--;
        }
}

/*
 * Free a process id - called from proc_free (in kern_exit.c)
 *
 * Called with the proc_lock held.
 */
void
proc_free_pid(pid_t pid)
{

        KASSERT(mutex_owned(&proc_lock));
        proc_free_pid_internal(pid, PT_F_PROC);
}

/*
 * Free a process id used by an LWP.  If this was the process's
 * first LWP, we convert the slot to point to the process; the
 * entry will get cleaned up later when the process finishes exiting.
 *
 * If not, then it's the same as proc_free_pid().
 */
void
proc_free_lwpid(struct proc *p, pid_t pid)
{

        KASSERT(mutex_owned(&proc_lock));

        if (__predict_true(p->p_pid == pid)) {
                struct pid_table *pt;

                pt = &pid_table[pid & pid_tbl_mask];

                KASSERT(pt->pt_pid == pid);
                KASSERT(PT_IS_LWP(pt->pt_slot));
                KASSERT(PT_GET_LWP(pt->pt_slot)->l_proc == p);

                pt->pt_slot = PT_SET_PROC(p);
                return;
        }
        proc_free_pid_internal(pid, PT_F_LWP);
}

void
proc_free_mem(struct proc *p)
{

        kdtrace_proc_dtor(NULL, p);
        pool_cache_put(proc_cache, p);
}

/*
 * proc_enterpgrp: move p to a new or existing process group (and session).
 *
 * If we are creating a new pgrp, the pgid should equal
 * the calling process' pid.
 * If is only valid to enter a process group that is in the session
 * of the process.
 * Also mksess should only be set if we are creating a process group
 *
 * Only called from sys_setsid, sys_setpgid and posix_spawn/spawn_return.
 */
int
proc_enterpgrp(struct proc *curp, pid_t pid, pid_t pgid, bool mksess)
{
        struct pgrp *new_pgrp, *pgrp;
        struct session *sess;
        struct proc *p;
        int rval;
        pid_t pg_id = NO_PGID;

        /* Allocate data areas we might need before doing any validity checks */
        sess = mksess ? kmem_alloc(sizeof(*sess), KM_SLEEP) : NULL;
        new_pgrp = kmem_alloc(sizeof(*new_pgrp), KM_SLEEP);

        mutex_enter(&proc_lock);

        /* Check pgrp exists or can be created */
        pgrp = pid_table[pgid & pid_tbl_mask].pt_pgrp;
        if (pgrp != NULL && pgrp->pg_id != pgid)
                goto eperm;

        /* Can only set another process under restricted circumstances. */
        if (pid != curp->p_pid) {
                /* Must exist and be one of our children... */
                p = proc_find_internal(pid, false);
                if (p == NULL || !p_inferior(p, curp)) {
                        rval = SET_ERROR(ESRCH);
                        goto done;
                }
                /* ... in the same session... */
                if (sess != NULL || p->p_session != curp->p_session)
                        goto eperm;
                /* ... existing pgid must be in same session ... */
                if (pgrp != NULL && pgrp->pg_session != p->p_session)
                        goto eperm;
                /* ... and not done an exec. */
                if (p->p_flag & PK_EXEC) {
                        rval = SET_ERROR(EACCES);
                        goto done;
                }
        } else {
                /* ... setsid() cannot re-enter a pgrp */
                if (mksess && (curp->p_pgid == curp->p_pid ||
                    pgrp_find(curp->p_pid)))
                        goto eperm;
                p = curp;
        }

        /* Changing the process group/session of a session
           leader is definitely off limits. */
        if (SESS_LEADER(p)) {
                if (sess == NULL && p->p_pgrp == pgrp) {
                        /* unless it's a definite noop */
                        rval = 0;
                        goto done;
                }
                goto eperm;
        }

        /* Can only create a process group with id of process */
        if (pgrp == NULL && pgid != pid)
                goto eperm;

        /* Can only create a session if creating pgrp */
        if (sess != NULL && pgrp != NULL)
                goto eperm;

        /* Check we allocated memory for a pgrp... */
        if (pgrp == NULL && new_pgrp == NULL)
                goto eperm;

        /* Don't attach to 'zombie' pgrp */
        if (pgrp != NULL && LIST_EMPTY(&pgrp->pg_members))
                goto eperm;

        /* Expect to succeed now */
        rval = 0;

        if (pgrp == p->p_pgrp)
                /* nothing to do */
                goto done;

        /* Ok all setup, link up required structures */

        if (pgrp == NULL) {
                pgrp = new_pgrp;
                new_pgrp = NULL;
                if (sess != NULL) {
                        sess->s_sid = p->p_pid;
                        sess->s_leader = p;
                        sess->s_count = 1;
                        sess->s_ttyvp = NULL;
                        sess->s_ttyp = NULL;
                        sess->s_flags = p->p_session->s_flags & ~S_LOGIN_SET;
                        memcpy(sess->s_login, p->p_session->s_login,
                            sizeof(sess->s_login));
                        p->p_lflag &= ~PL_CONTROLT;
                } else {
                        sess = p->p_pgrp->pg_session;
                        proc_sesshold(sess);
                }
                pgrp->pg_session = sess;
                sess = NULL;

                pgrp->pg_id = pgid;
                LIST_INIT(&pgrp->pg_members);
#ifdef DIAGNOSTIC
                if (__predict_false(pid_table[pgid & pid_tbl_mask].pt_pgrp))
                        panic("enterpgrp: pgrp table slot in use");
                if (__predict_false(mksess && p != curp))
                        panic("enterpgrp: mksession and p != curproc");
#endif
                pid_table[pgid & pid_tbl_mask].pt_pgrp = pgrp;
                pgrp->pg_jobc = 0;
        }

        /*
         * Adjust eligibility of affected pgrps to participate in job control.
         * Increment eligibility counts before decrementing, otherwise we
         * could reach 0 spuriously during the first call.
         */
        fixjobc(p, pgrp, 1);
        fixjobc(p, p->p_pgrp, 0);

        /* Interlock with ttread(). */
        mutex_spin_enter(&tty_lock);

        /* Move process to requested group. */
        LIST_REMOVE(p, p_pglist);
        if (LIST_EMPTY(&p->p_pgrp->pg_members))
                /* defer delete until we've dumped the lock */
                pg_id = p->p_pgrp->pg_id;
        p->p_pgrp = pgrp;
        LIST_INSERT_HEAD(&pgrp->pg_members, p, p_pglist);

        /* Done with the swap; we can release the tty mutex. */
        mutex_spin_exit(&tty_lock);
        goto done;

eperm:
        rval = SET_ERROR(EPERM);
done:
        if (pg_id != NO_PGID) {
                /* Releases proc_lock. */
                pg_delete(pg_id);
        } else {
                mutex_exit(&proc_lock);
        }
        if (sess != NULL)
                kmem_free(sess, sizeof(*sess));
        if (new_pgrp != NULL)
                kmem_free(new_pgrp, sizeof(*new_pgrp));
#ifdef DEBUG_PGRP
        if (__predict_false(rval))
                printf("enterpgrp(%d,%d,%d), curproc %d, rval %d\n",
                        pid, pgid, mksess, curp->p_pid, rval);
#endif
        return rval;
}

/*
 * proc_leavepgrp: remove a process from its process group.
 *  => must be called with the proc_lock held, which will be released;
 */
void
proc_leavepgrp(struct proc *p)
{
        struct pgrp *pgrp;

        KASSERT(mutex_owned(&proc_lock));

        /* Interlock with ttread() */
        mutex_spin_enter(&tty_lock);
        pgrp = p->p_pgrp;
        LIST_REMOVE(p, p_pglist);
        p->p_pgrp = NULL;
        mutex_spin_exit(&tty_lock);

        if (LIST_EMPTY(&pgrp->pg_members)) {
                /* Releases proc_lock. */
                pg_delete(pgrp->pg_id);
        } else {
                mutex_exit(&proc_lock);
        }
}

/*
 * pg_remove: remove a process group from the table.
 *  => must be called with the proc_lock held;
 *  => returns process group to free;
 */
static struct pgrp *
pg_remove(pid_t pg_id)
{
        struct pgrp *pgrp;
        struct pid_table *pt;

        KASSERT(mutex_owned(&proc_lock));

        pt = &pid_table[pg_id & pid_tbl_mask];
        pgrp = pt->pt_pgrp;

        KASSERT(pgrp != NULL);
        KASSERT(pgrp->pg_id == pg_id);
        KASSERT(LIST_EMPTY(&pgrp->pg_members));

        pt->pt_pgrp = NULL;

        if (!PT_VALID(pt->pt_slot)) {
                /* Orphaned pgrp, put slot onto free list. */
                KASSERT((PT_NEXT(pt->pt_slot) & pid_tbl_mask) == 0);
                pg_id &= pid_tbl_mask;
                pt = &pid_table[last_free_pt];
                pt->pt_slot = PT_SET_FREE(PT_NEXT(pt->pt_slot) | pg_id);
                KASSERT(pt->pt_pid == 0);
                last_free_pt = pg_id;
                pid_alloc_cnt--;
        }
        return pgrp;
}

/*
 * pg_delete: delete and free a process group.
 *  => must be called with the proc_lock held, which will be released.
 */
static void
pg_delete(pid_t pg_id)
{
        struct pgrp *pg;
        struct tty *ttyp;
        struct session *ss;

        KASSERT(mutex_owned(&proc_lock));

        pg = pid_table[pg_id & pid_tbl_mask].pt_pgrp;
        if (pg == NULL || pg->pg_id != pg_id || !LIST_EMPTY(&pg->pg_members)) {
                mutex_exit(&proc_lock);
                return;
        }

        ss = pg->pg_session;

        /* Remove reference (if any) from tty to this process group */
        mutex_spin_enter(&tty_lock);
        ttyp = ss->s_ttyp;
        if (ttyp != NULL && ttyp->t_pgrp == pg) {
                ttyp->t_pgrp = NULL;
                KASSERT(ttyp->t_session == ss);
        }
        mutex_spin_exit(&tty_lock);

        /*
         * The leading process group in a session is freed by proc_sessrele(),
         * if last reference.  It will also release the locks.
         */
        pg = (ss->s_sid != pg->pg_id) ? pg_remove(pg_id) : NULL;
        proc_sessrele(ss);

        if (pg != NULL) {
                /* Free it, if was not done above. */
                kmem_free(pg, sizeof(struct pgrp));
        }
}

/*
 * Adjust pgrp jobc counters when specified process changes process group.
 * We count the number of processes in each process group that "qualify"
 * the group for terminal job control (those with a parent in a different
 * process group of the same session).  If that count reaches zero, the
 * process group becomes orphaned.  Check both the specified process'
 * process group and that of its children.
 * entering == 0 => p is leaving specified group.
 * entering == 1 => p is entering specified group.
 *
 * Call with proc_lock held.
 */
void
fixjobc(struct proc *p, struct pgrp *pgrp, int entering)
{
        struct pgrp *hispgrp;
        struct session *mysession = pgrp->pg_session;
        struct proc *child;

        KASSERT(mutex_owned(&proc_lock));

        /*
         * Check p's parent to see whether p qualifies its own process
         * group; if so, adjust count for p's process group.
         */
        hispgrp = p->p_pptr->p_pgrp;
        if (hispgrp != pgrp && hispgrp->pg_session == mysession) {
                if (entering) {
                        pgrp->pg_jobc++;
                        p->p_lflag &= ~PL_ORPHANPG;
                } else {
                        /* KASSERT(pgrp->pg_jobc > 0); */
                        if (--pgrp->pg_jobc == 0)
                                orphanpg(pgrp);
                }
        }

        /*
         * Check this process' children to see whether they qualify
         * their process groups; if so, adjust counts for children's
         * process groups.
         */
        LIST_FOREACH(child, &p->p_children, p_sibling) {
                hispgrp = child->p_pgrp;
                if (hispgrp != pgrp && hispgrp->pg_session == mysession &&
                    !P_ZOMBIE(child)) {
                        if (entering) {
                                child->p_lflag &= ~PL_ORPHANPG;
                                hispgrp->pg_jobc++;
                        } else {
                                /* KASSERT(hispgrp->pg_jobc > 0); */
                                if (--hispgrp->pg_jobc == 0)
                                        orphanpg(hispgrp);
                        }
                }
        }
}

/*
 * A process group has become orphaned;
 * if there are any stopped processes in the group,
 * hang-up all process in that group.
 *
 * Call with proc_lock held.
 */
static void
orphanpg(struct pgrp *pg)
{
        struct proc *p;

        KASSERT(mutex_owned(&proc_lock));

        LIST_FOREACH(p, &pg->pg_members, p_pglist) {
                if (p->p_stat == SSTOP) {
                        p->p_lflag |= PL_ORPHANPG;
                        psignal(p, SIGHUP);
                        psignal(p, SIGCONT);
                }
        }
}

#ifdef DDB
#include <ddb/db_output.h>
void pidtbl_dump(void);
void
pidtbl_dump(void)
{
        struct pid_table *pt;
        struct proc *p;
        struct pgrp *pgrp;
        uintptr_t slot;
        int id;

        db_printf("pid table %p size %x, next %x, last %x\n",
                pid_table, pid_tbl_mask+1,
                next_free_pt, last_free_pt);
        for (pt = pid_table, id = 0; id <= pid_tbl_mask; id++, pt++) {
                slot = pt->pt_slot;
                if (!PT_VALID(slot) && !pt->pt_pgrp)
                        continue;
                if (PT_IS_LWP(slot)) {
                        p = PT_GET_LWP(slot)->l_proc;
                } else if (PT_IS_PROC(slot)) {
                        p = PT_GET_PROC(slot);
                } else {
                        p = NULL;
                }
                db_printf("  id %x: ", id);
                if (p != NULL)
                        db_printf("slotpid %d proc %p id %d (0x%x) %s\n",
                                pt->pt_pid, p, p->p_pid, p->p_pid, p->p_comm);
                else
                        db_printf("next %x use %x\n",
                                PT_NEXT(slot) & pid_tbl_mask,
                                PT_NEXT(slot) & ~pid_tbl_mask);
                if ((pgrp = pt->pt_pgrp)) {
                        db_printf("\tsession %p, sid %d, count %d, login %s\n",
                            pgrp->pg_session, pgrp->pg_session->s_sid,
                            pgrp->pg_session->s_count,
                            pgrp->pg_session->s_login);
                        db_printf("\tpgrp %p, pg_id %d, pg_jobc %d, members %p\n",
                            pgrp, pgrp->pg_id, pgrp->pg_jobc,
                            LIST_FIRST(&pgrp->pg_members));
                        LIST_FOREACH(p, &pgrp->pg_members, p_pglist) {
                                db_printf("\t\tpid %d addr %p pgrp %p %s\n",
                                    p->p_pid, p, p->p_pgrp, p->p_comm);
                        }
                }
        }
}
#endif /* DDB */

#ifdef KSTACK_CHECK_MAGIC

#define KSTACK_MAGIC    0xdeadbeaf

/* XXX should be per process basis? */
static int      kstackleftmin = KSTACK_SIZE;
static int      kstackleftthres = KSTACK_SIZE / 8;

void
kstack_setup_magic(const struct lwp *l)
{
        uint32_t *ip;
        uint32_t const *end;

        KASSERT(l != NULL);
        KASSERT(l != &lwp0);

        /*
         * fill all the stack with magic number
         * so that later modification on it can be detected.
         */
        ip = (uint32_t *)KSTACK_LOWEST_ADDR(l);
        end = (uint32_t *)((char *)KSTACK_LOWEST_ADDR(l) + KSTACK_SIZE);
        for (; ip < end; ip++) {
                *ip = KSTACK_MAGIC;
        }
}

void
kstack_check_magic(const struct lwp *l)
{
        uint32_t const *ip, *end;
        int stackleft;

        KASSERT(l != NULL);

        /* don't check proc0 */ /*XXX*/
        if (l == &lwp0)
                return;

#ifdef __MACHINE_STACK_GROWS_UP
        /* stack grows upwards (eg. hppa) */
        ip = (uint32_t *)((void *)KSTACK_LOWEST_ADDR(l) + KSTACK_SIZE);
        end = (uint32_t *)KSTACK_LOWEST_ADDR(l);
        for (ip--; ip >= end; ip--)
                if (*ip != KSTACK_MAGIC)
                        break;

        stackleft = (void *)KSTACK_LOWEST_ADDR(l) + KSTACK_SIZE - (void *)ip;
#else /* __MACHINE_STACK_GROWS_UP */
        /* stack grows downwards (eg. i386) */
        ip = (uint32_t *)KSTACK_LOWEST_ADDR(l);
        end = (uint32_t *)((char *)KSTACK_LOWEST_ADDR(l) + KSTACK_SIZE);
        for (; ip < end; ip++)
                if (*ip != KSTACK_MAGIC)
                        break;

        stackleft = ((const char *)ip) - (const char *)KSTACK_LOWEST_ADDR(l);
#endif /* __MACHINE_STACK_GROWS_UP */

        if (kstackleftmin > stackleft) {
                kstackleftmin = stackleft;
                if (stackleft < kstackleftthres)
                        printf("warning: kernel stack left %d bytes"
                            "(pid %u:lid %u)\n", stackleft,
                            (u_int)l->l_proc->p_pid, (u_int)l->l_lid);
        }

        if (stackleft <= 0) {
                panic("magic on the top of kernel stack changed for "
                    "pid %u, lid %u: maybe kernel stack overflow",
                    (u_int)l->l_proc->p_pid, (u_int)l->l_lid);
        }
}
#endif /* KSTACK_CHECK_MAGIC */

int
proclist_foreach_call(struct proclist *list,
    int (*callback)(struct proc *, void *arg), void *arg)
{
        struct proc marker;
        struct proc *p;
        int ret = 0;

        marker.p_flag = PK_MARKER;
        mutex_enter(&proc_lock);
        for (p = LIST_FIRST(list); ret == 0 && p != NULL;) {
                if (p->p_flag & PK_MARKER) {
                        p = LIST_NEXT(p, p_list);
                        continue;
                }
                LIST_INSERT_AFTER(p, &marker, p_list);
                ret = (*callback)(p, arg);
                KASSERT(mutex_owned(&proc_lock));
                p = LIST_NEXT(&marker, p_list);
                LIST_REMOVE(&marker, p_list);
        }
        mutex_exit(&proc_lock);

        return ret;
}

int
proc_vmspace_getref(struct proc *p, struct vmspace **vm)
{

        /* XXXCDC: how should locking work here? */

        /* curproc exception is for coredump. */

        if ((p != curproc && (p->p_sflag & PS_WEXIT) != 0) ||
            (p->p_vmspace->vm_refcnt < 1)) {
                return SET_ERROR(EFAULT);
        }

        uvmspace_addref(p->p_vmspace);
        *vm = p->p_vmspace;

        return 0;
}

/*
 * Acquire a write lock on the process credential.
 */
void
proc_crmod_enter(void)
{
        struct lwp *l = curlwp;
        struct proc *p = l->l_proc;
        kauth_cred_t oc;

        /* Reset what needs to be reset in plimit. */
        if (p->p_limit->pl_corename != defcorename) {
                lim_setcorename(p, defcorename, 0);
        }

        mutex_enter(p->p_lock);

        /* Ensure the LWP cached credentials are up to date. */
        if ((oc = l->l_cred) != p->p_cred) {
                l->l_cred = kauth_cred_hold(p->p_cred);
                kauth_cred_free(oc);
        }
}

/*
 * Set in a new process credential, and drop the write lock.  The credential
 * must have a reference already.  Optionally, free a no-longer required
 * credential.
 */
void
proc_crmod_leave(kauth_cred_t scred, kauth_cred_t fcred, bool sugid)
{
        struct lwp *l = curlwp, *l2;
        struct proc *p = l->l_proc;
        kauth_cred_t oc;

        KASSERT(mutex_owned(p->p_lock));

        /* Is there a new credential to set in? */
        if (scred != NULL) {
                p->p_cred = scred;
                LIST_FOREACH(l2, &p->p_lwps, l_sibling) {
                        if (l2 != l) {
                                lwp_lock(l2);
                                l2->l_flag |= LW_CACHECRED;
                                lwp_need_userret(l2);
                                lwp_unlock(l2);
                        }
                }

                /* Ensure the LWP cached credentials are up to date. */
                if ((oc = l->l_cred) != scred) {
                        l->l_cred = kauth_cred_hold(scred);
                }
        } else
                oc = NULL;      /* XXXgcc */

        if (sugid) {
                /*
                 * Mark process as having changed credentials, stops
                 * tracing etc.
                 */
                p->p_flag |= PK_SUGID;
        }

        mutex_exit(p->p_lock);

        /* If there is a credential to be released, free it now. */
        if (fcred != NULL) {
                KASSERT(scred != NULL);
                kauth_cred_free(fcred);
                if (oc != scred)
                        kauth_cred_free(oc);
        }
}

/*
 * proc_specific_key_create --
 *      Create a key for subsystem proc-specific data.
 */
int
proc_specific_key_create(specificdata_key_t *keyp, specificdata_dtor_t dtor)
{

        return (specificdata_key_create(proc_specificdata_domain, keyp, dtor));
}

/*
 * proc_specific_key_delete --
 *      Delete a key for subsystem proc-specific data.
 */
void
proc_specific_key_delete(specificdata_key_t key)
{

        specificdata_key_delete(proc_specificdata_domain, key);
}

/*
 * proc_initspecific --
 *      Initialize a proc's specificdata container.
 */
void
proc_initspecific(struct proc *p)
{
        int error __diagused;

        error = specificdata_init(proc_specificdata_domain, &p->p_specdataref);
        KASSERT(error == 0);
}

/*
 * proc_finispecific --
 *      Finalize a proc's specificdata container.
 */
void
proc_finispecific(struct proc *p)
{

        specificdata_fini(proc_specificdata_domain, &p->p_specdataref);
}

/*
 * proc_getspecific --
 *      Return proc-specific data corresponding to the specified key.
 */
void *
proc_getspecific(struct proc *p, specificdata_key_t key)
{

        return (specificdata_getspecific(proc_specificdata_domain,
                                         &p->p_specdataref, key));
}

/*
 * proc_setspecific --
 *      Set proc-specific data corresponding to the specified key.
 */
void
proc_setspecific(struct proc *p, specificdata_key_t key, void *data)
{

        specificdata_setspecific(proc_specificdata_domain,
                                 &p->p_specdataref, key, data);
}

int
proc_uidmatch(kauth_cred_t cred, kauth_cred_t target)
{

        if (kauth_cred_getuid(cred) != kauth_cred_getuid(target) ||
            kauth_cred_getuid(cred) != kauth_cred_getsvuid(target)) {
                /*
                 * suid proc of ours or proc not ours
                 */
                return SET_ERROR(EPERM);
        } else if (kauth_cred_getgid(target) != kauth_cred_getsvgid(target)) {
                /*
                 * sgid proc has sgid back to us temporarily
                 */
                return SET_ERROR(EPERM);
        } else {
                /*
                 * our rgid must be in target's group list (ie,
                 * sub-processes started by a sgid process)
                 */
                int ismember = 0;

                if (kauth_cred_ismember_gid(cred,
                    kauth_cred_getgid(target), &ismember) != 0 ||
                    !ismember)
                        return SET_ERROR(EPERM);
        }

        return 0;
}

/*
 * sysctl stuff
 */

#define KERN_PROCSLOP   (5 * sizeof(struct kinfo_proc))

static const u_int sysctl_flagmap[] = {
        PK_ADVLOCK, P_ADVLOCK,
        PK_EXEC, P_EXEC,
        PK_NOCLDWAIT, P_NOCLDWAIT,
        PK_32, P_32,
        PK_CLDSIGIGN, P_CLDSIGIGN,
        PK_SUGID, P_SUGID,
        0
};

static const u_int sysctl_sflagmap[] = {
        PS_NOCLDSTOP, P_NOCLDSTOP,
        PS_WEXIT, P_WEXIT,
        PS_STOPFORK, P_STOPFORK,
        PS_STOPEXEC, P_STOPEXEC,
        PS_STOPEXIT, P_STOPEXIT,
        0
};

static const u_int sysctl_slflagmap[] = {
        PSL_TRACED, P_TRACED,
        PSL_CHTRACED, P_CHTRACED,
        PSL_SYSCALL, P_SYSCALL,
        0
};

static const u_int sysctl_lflagmap[] = {
        PL_CONTROLT, P_CONTROLT,
        PL_PPWAIT, P_PPWAIT,
        0
};

static const u_int sysctl_stflagmap[] = {
        PST_PROFIL, P_PROFIL,
        0

};

/* used by kern_lwp also */
const u_int sysctl_lwpflagmap[] = {
        LW_SINTR, L_SINTR,
        LW_SYSTEM, L_SYSTEM,
        0
};

/*
 * Find the most ``active'' lwp of a process and return it for ps display
 * purposes
 */
static struct lwp *
proc_active_lwp(struct proc *p)
{
        static const int ostat[] = {
                0,
                2,      /* LSIDL */
                6,      /* LSRUN */
                5,      /* LSSLEEP */
                4,      /* LSSTOP */
                0,      /* LSZOMB */
                1,      /* LSDEAD */
                7,      /* LSONPROC */
                3       /* LSSUSPENDED */
        };

        struct lwp *l, *lp = NULL;
        LIST_FOREACH(l, &p->p_lwps, l_sibling) {
                KASSERT(l->l_stat >= 0);
                KASSERT(l->l_stat < __arraycount(ostat));
                if (lp == NULL ||
                    ostat[l->l_stat] > ostat[lp->l_stat] ||
                    (ostat[l->l_stat] == ostat[lp->l_stat] &&
                    l->l_cpticks > lp->l_cpticks)) {
                        lp = l;
                        continue;
                }
        }
        return lp;
}

static int
sysctl_doeproc(SYSCTLFN_ARGS)
{
        union {
                struct kinfo_proc kproc;
                struct kinfo_proc2 kproc2;
        } *kbuf;
        struct proc *p, *next, *marker;
        char *where, *dp;
        int type, op, arg, error;
        u_int elem_size, kelem_size, elem_count;
        size_t buflen, needed;
        bool match, zombie, mmmbrains;
        const bool allowaddr = get_expose_address(curproc);

        if (namelen == 1 && name[0] == CTL_QUERY)
                return (sysctl_query(SYSCTLFN_CALL(rnode)));

        dp = where = oldp;
        buflen = where != NULL ? *oldlenp : 0;
        error = 0;
        needed = 0;
        type = rnode->sysctl_num;

        if (type == KERN_PROC) {
                if (namelen == 0)
                        return SET_ERROR(EINVAL);
                switch (op = name[0]) {
                case KERN_PROC_ALL:
                        /*
                         * note: kvm_getprocs(KERN_PROC_ALL) gives
                         * us namelen=2. ignore the extra argument,
                         * which is typically 0.
                         */
                        if (namelen != 1 && namelen != 2)
                                return SET_ERROR(EINVAL);
                        arg = 0;
                        break;
                default:
                        if (namelen != 2)
                                return SET_ERROR(EINVAL);
                        arg = name[1];
                        break;
                }
                elem_count = 0; /* Hush little compiler, don't you cry */
                kelem_size = elem_size = sizeof(kbuf->kproc);
        } else {
                if (namelen != 4)
                        return SET_ERROR(EINVAL);
                op = name[0];
                arg = name[1];
                elem_size = name[2];
                elem_count = name[3];
                kelem_size = sizeof(kbuf->kproc2);
        }

        sysctl_unlock();

        kbuf = kmem_zalloc(sizeof(*kbuf), KM_SLEEP);
        marker = kmem_alloc(sizeof(*marker), KM_SLEEP);
        marker->p_flag = PK_MARKER;

        mutex_enter(&proc_lock);
        /*
         * Start with zombies to prevent reporting processes twice, in case they
         * are dying and being moved from the list of alive processes to zombies.
         */
        mmmbrains = true;
        for (p = LIST_FIRST(&zombproc);; p = next) {
                if (p == NULL) {
                        if (mmmbrains) {
                                p = LIST_FIRST(&allproc);
                                mmmbrains = false;
                        }
                        if (p == NULL)
                                break;
                }
                next = LIST_NEXT(p, p_list);
                if ((p->p_flag & PK_MARKER) != 0)
                        continue;

                /*
                 * Skip embryonic processes.
                 */
                if (p->p_stat == SIDL)
                        continue;

                mutex_enter(p->p_lock);
                error = kauth_authorize_process(l->l_cred,
                    KAUTH_PROCESS_CANSEE, p,
                    KAUTH_ARG(KAUTH_REQ_PROCESS_CANSEE_EPROC), NULL, NULL);
                if (error != 0) {
                        mutex_exit(p->p_lock);
                        continue;
                }

                /*
                 * Handle all the operations in one switch on the cost of
                 * algorithm complexity is on purpose. The win splitting this
                 * function into several similar copies makes maintenance
                 * burden, code grow and boost is negligible in practical
                 * systems.
                 */
                switch (op) {
                case KERN_PROC_PID:
                        match = (p->p_pid == (pid_t)arg);
                        break;

                case KERN_PROC_PGRP:
                        match = (p->p_pgrp->pg_id == (pid_t)arg);
                        break;

                case KERN_PROC_SESSION:
                        match = (p->p_session->s_sid == (pid_t)arg);
                        break;

                case KERN_PROC_TTY:
                        match = true;
                        if (arg == (int) KERN_PROC_TTY_REVOKE) {
                                if ((p->p_lflag & PL_CONTROLT) == 0 ||
                                    p->p_session->s_ttyp == NULL ||
                                    p->p_session->s_ttyvp != NULL) {
                                        match = false;
                                }
                        } else if ((p->p_lflag & PL_CONTROLT) == 0 ||
                            p->p_session->s_ttyp == NULL) {
                                if ((dev_t)arg != KERN_PROC_TTY_NODEV) {
                                        match = false;
                                }
                        } else if (p->p_session->s_ttyp->t_dev != (dev_t)arg) {
                                match = false;
                        }
                        break;

                case KERN_PROC_UID:
                        match = (kauth_cred_geteuid(p->p_cred) == (uid_t)arg);
                        break;

                case KERN_PROC_RUID:
                        match = (kauth_cred_getuid(p->p_cred) == (uid_t)arg);
                        break;

                case KERN_PROC_GID:
                        match = (kauth_cred_getegid(p->p_cred) == (uid_t)arg);
                        break;

                case KERN_PROC_RGID:
                        match = (kauth_cred_getgid(p->p_cred) == (uid_t)arg);
                        break;

                case KERN_PROC_ALL:
                        match = true;
                        /* allow everything */
                        break;

                default:
                        error = SET_ERROR(EINVAL);
                        mutex_exit(p->p_lock);
                        goto cleanup;
                }
                if (!match) {
                        mutex_exit(p->p_lock);
                        continue;
                }

                /*
                 * Grab a hold on the process.
                 */
                if (mmmbrains) {
                        zombie = true;
                } else {
                        zombie = !rw_tryenter(&p->p_reflock, RW_READER);
                }
                if (zombie) {
                        LIST_INSERT_AFTER(p, marker, p_list);
                }

                if (buflen >= elem_size &&
                    (type == KERN_PROC || elem_count > 0)) {
                        ruspace(p);     /* Update process vm resource use */

                        if (type == KERN_PROC) {
                                fill_proc(p, &kbuf->kproc.kp_proc, allowaddr);
                                fill_eproc(p, &kbuf->kproc.kp_eproc, zombie,
                                    allowaddr);
                        } else {
                                fill_kproc2(p, &kbuf->kproc2, zombie,
                                    allowaddr);
                                elem_count--;
                        }
                        mutex_exit(p->p_lock);
                        mutex_exit(&proc_lock);
                        /*
                         * Copy out elem_size, but not larger than kelem_size
                         */
                        error = sysctl_copyout(l, kbuf, dp,
                            uimin(kelem_size, elem_size));
                        mutex_enter(&proc_lock);
                        if (error) {
                                goto bah;
                        }
                        dp += elem_size;
                        buflen -= elem_size;
                } else {
                        mutex_exit(p->p_lock);
                }
                needed += elem_size;

                /*
                 * Release reference to process.
                 */
                if (zombie) {
                        next = LIST_NEXT(marker, p_list);
                        LIST_REMOVE(marker, p_list);
                } else {
                        rw_exit(&p->p_reflock);
                        next = LIST_NEXT(p, p_list);
                }

                /*
                 * Short-circuit break quickly!
                 */
                if (op == KERN_PROC_PID)
                        break;
        }
        mutex_exit(&proc_lock);

        if (where != NULL) {
                *oldlenp = dp - where;
                if (needed > *oldlenp) {
                        error = SET_ERROR(ENOMEM);
                        goto out;
                }
        } else {
                needed += KERN_PROCSLOP;
                *oldlenp = needed;
        }
        kmem_free(kbuf, sizeof(*kbuf));
        kmem_free(marker, sizeof(*marker));
        sysctl_relock();
        return 0;
 bah:
        if (zombie)
                LIST_REMOVE(marker, p_list);
        else
                rw_exit(&p->p_reflock);
 cleanup:
        mutex_exit(&proc_lock);
 out:
        kmem_free(kbuf, sizeof(*kbuf));
        kmem_free(marker, sizeof(*marker));
        sysctl_relock();
        return error;
}

int
copyin_psstrings(struct proc *p, struct ps_strings *arginfo)
{
#if !defined(_RUMPKERNEL)
        int retval;

        if (p->p_flag & PK_32) {
                MODULE_HOOK_CALL(kern_proc32_copyin_hook, (p, arginfo),
                    enosys(), retval);
                return retval;
        }
#endif /* !defined(_RUMPKERNEL) */

        return copyin_proc(p, (void *)p->p_psstrp, arginfo, sizeof(*arginfo));
}

static int
copy_procargs_sysctl_cb(void *cookie_, const void *src, size_t off, size_t len)
{
        void **cookie = cookie_;
        struct lwp *l = cookie[0];
        char *dst = cookie[1];

        return sysctl_copyout(l, src, dst + off, len);
}

/*
 * sysctl helper routine for kern.proc_args pseudo-subtree.
 */
static int
sysctl_kern_proc_args(SYSCTLFN_ARGS)
{
        struct ps_strings pss;
        struct proc *p;
        pid_t pid;
        int type, error;
        void *cookie[2];

        if (namelen == 1 && name[0] == CTL_QUERY)
                return (sysctl_query(SYSCTLFN_CALL(rnode)));

        if (newp != NULL || namelen != 2)
                return SET_ERROR(EINVAL);
        pid = name[0];
        type = name[1];

        switch (type) {
        case KERN_PROC_PATHNAME:
                sysctl_unlock();
                error = fill_pathname(l, pid, oldp, oldlenp);
                sysctl_relock();
                return error;

        case KERN_PROC_CWD:
                sysctl_unlock();
                error = fill_cwd(l, pid, oldp, oldlenp);
                sysctl_relock();
                return error;

        case KERN_PROC_ARGV:
        case KERN_PROC_NARGV:
        case KERN_PROC_ENV:
        case KERN_PROC_NENV:
                /* ok */
                break;
        default:
                return SET_ERROR(EINVAL);
        }

        sysctl_unlock();

        /* check pid */
        mutex_enter(&proc_lock);
        if ((p = proc_find(pid)) == NULL) {
                error = SET_ERROR(EINVAL);
                goto out_locked;
        }
        mutex_enter(p->p_lock);

        /* Check permission. */
        if (type == KERN_PROC_ARGV || type == KERN_PROC_NARGV)
                error = kauth_authorize_process(l->l_cred, KAUTH_PROCESS_CANSEE,
                    p, KAUTH_ARG(KAUTH_REQ_PROCESS_CANSEE_ARGS), NULL, NULL);
        else if (type == KERN_PROC_ENV || type == KERN_PROC_NENV)
                error = kauth_authorize_process(l->l_cred, KAUTH_PROCESS_CANSEE,
                    p, KAUTH_ARG(KAUTH_REQ_PROCESS_CANSEE_ENV), NULL, NULL);
        else
                error = SET_ERROR(EINVAL); /* XXXGCC */
        if (error) {
                mutex_exit(p->p_lock);
                goto out_locked;
        }

        if (oldp == NULL) {
                if (type == KERN_PROC_NARGV || type == KERN_PROC_NENV)
                        *oldlenp = sizeof (int);
                else
                        *oldlenp = ARG_MAX;     /* XXX XXX XXX */
                error = 0;
                mutex_exit(p->p_lock);
                goto out_locked;
        }

        /*
         * Zombies don't have a stack, so we can't read their psstrings.
         * System processes also don't have a user stack.
         */
        if (P_ZOMBIE(p) || (p->p_flag & PK_SYSTEM) != 0) {
                error = SET_ERROR(EINVAL);
                mutex_exit(p->p_lock);
                goto out_locked;
        }

        error = rw_tryenter(&p->p_reflock, RW_READER) ? 0 : SET_ERROR(EBUSY);
        mutex_exit(p->p_lock);
        if (error) {
                goto out_locked;
        }
        mutex_exit(&proc_lock);

        if (type == KERN_PROC_NARGV || type == KERN_PROC_NENV) {
                int value;
                if ((error = copyin_psstrings(p, &pss)) == 0) {
                        if (type == KERN_PROC_NARGV)
                                value = pss.ps_nargvstr;
                        else
                                value = pss.ps_nenvstr;
                        error = sysctl_copyout(l, &value, oldp, sizeof(value));
                        *oldlenp = sizeof(value);
                }
        } else {
                cookie[0] = l;
                cookie[1] = oldp;
                error = copy_procargs(p, type, oldlenp,
                    copy_procargs_sysctl_cb, cookie);
        }
        rw_exit(&p->p_reflock);
        sysctl_relock();
        return error;

out_locked:
        mutex_exit(&proc_lock);
        sysctl_relock();
        return error;
}

int
copy_procargs(struct proc *p, int oid, size_t *limit,
    int (*cb)(void *, const void *, size_t, size_t), void *cookie)
{
        struct ps_strings pss;
        size_t len, i, loaded, entry_len;
        struct uio auio;
        struct iovec aiov;
        int error, argvlen;
        char *arg;
        char **argv;
        vaddr_t user_argv;
        struct vmspace *vmspace;

        /*
         * Allocate a temporary buffer to hold the argument vector and
         * the arguments themselve.
         */
        arg = kmem_alloc(PAGE_SIZE, KM_SLEEP);
        argv = kmem_alloc(PAGE_SIZE, KM_SLEEP);

        /*
         * Lock the process down in memory.
         */
        vmspace = p->p_vmspace;
        uvmspace_addref(vmspace);

        /*
         * Read in the ps_strings structure.
         */
        if ((error = copyin_psstrings(p, &pss)) != 0)
                goto done;

        /*
         * Now read the address of the argument vector.
         */
        switch (oid) {
        case KERN_PROC_ARGV:
                user_argv = (uintptr_t)pss.ps_argvstr;
                argvlen = pss.ps_nargvstr;
                break;
        case KERN_PROC_ENV:
                user_argv = (uintptr_t)pss.ps_envstr;
                argvlen = pss.ps_nenvstr;
                break;
        default:
                error = SET_ERROR(EINVAL);
                goto done;
        }

        if (argvlen < 0) {
                error = SET_ERROR(EIO);
                goto done;
        }


        /*
         * Now copy each string.
         */
        len = 0; /* bytes written to user buffer */
        loaded = 0; /* bytes from argv already processed */
        i = 0; /* To make compiler happy */
        entry_len = PROC_PTRSZ(p);

        for (; argvlen; --argvlen) {
                int finished = 0;
                vaddr_t base;
                size_t xlen;
                int j;

                if (loaded == 0) {
                        size_t rem = entry_len * argvlen;
                        loaded = MIN(rem, PAGE_SIZE);
                        error = copyin_vmspace(vmspace,
                            (const void *)user_argv, argv, loaded);
                        if (error)
                                break;
                        user_argv += loaded;
                        i = 0;
                }

#if !defined(_RUMPKERNEL)
                if (p->p_flag & PK_32)
                        MODULE_HOOK_CALL(kern_proc32_base_hook,
                            (argv, i++), 0, base);
                else
#endif /* !defined(_RUMPKERNEL) */
                        base = (vaddr_t)argv[i++];
                loaded -= entry_len;

                /*
                 * The program has messed around with its arguments,
                 * possibly deleting some, and replacing them with
                 * NULL's. Treat this as the last argument and not
                 * a failure.
                 */
                if (base == 0)
                        break;

                while (!finished) {
                        xlen = PAGE_SIZE - (base & PAGE_MASK);

                        aiov.iov_base = arg;
                        aiov.iov_len = PAGE_SIZE;
                        auio.uio_iov = &aiov;
                        auio.uio_iovcnt = 1;
                        auio.uio_offset = base;
                        auio.uio_resid = xlen;
                        auio.uio_rw = UIO_READ;
                        UIO_SETUP_SYSSPACE(&auio);
                        error = uvm_io(&vmspace->vm_map, &auio, 0);
                        if (error)
                                goto done;

                        /* Look for the end of the string */
                        for (j = 0; j < xlen; j++) {
                                if (arg[j] == '\0') {
                                        xlen = j + 1;
                                        finished = 1;
                                        break;
                                }
                        }

                        /* Check for user buffer overflow */
                        if (len + xlen > *limit) {
                                finished = 1;
                                if (len > *limit)
                                        xlen = 0;
                                else
                                        xlen = *limit - len;
                        }

                        /* Copyout the page */
                        error = (*cb)(cookie, arg, len, xlen);
                        if (error)
                                goto done;

                        len += xlen;
                        base += xlen;
                }
        }
        *limit = len;

done:
        kmem_free(argv, PAGE_SIZE);
        kmem_free(arg, PAGE_SIZE);
        uvmspace_free(vmspace);
        return error;
}

/*
 * Fill in a proc structure for the specified process.
 */
static void
fill_proc(const struct proc *psrc, struct proc *p, bool allowaddr)
{
        COND_SET_STRUCT(p->p_list, psrc->p_list, allowaddr);
        memset(&p->p_auxlock, 0, sizeof(p->p_auxlock));
        COND_SET_STRUCT(p->p_lock, psrc->p_lock, allowaddr);
        memset(&p->p_stmutex, 0, sizeof(p->p_stmutex));
        memset(&p->p_reflock, 0, sizeof(p->p_reflock));
        COND_SET_STRUCT(p->p_waitcv, psrc->p_waitcv, allowaddr);
        COND_SET_STRUCT(p->p_lwpcv, psrc->p_lwpcv, allowaddr);
        COND_SET_PTR(p->p_cred, psrc->p_cred, allowaddr);
        COND_SET_PTR(p->p_fd, psrc->p_fd, allowaddr);
        COND_SET_PTR(p->p_cwdi, psrc->p_cwdi, allowaddr);
        COND_SET_PTR(p->p_stats, psrc->p_stats, allowaddr);
        COND_SET_PTR(p->p_limit, psrc->p_limit, allowaddr);
        COND_SET_PTR(p->p_vmspace, psrc->p_vmspace, allowaddr);
        COND_SET_PTR(p->p_sigacts, psrc->p_sigacts, allowaddr);
        COND_SET_PTR(p->p_aio, psrc->p_aio, allowaddr);
        p->p_mqueue_cnt = psrc->p_mqueue_cnt;
        memset(&p->p_specdataref, 0, sizeof(p->p_specdataref));
        p->p_exitsig = psrc->p_exitsig;
        p->p_flag = psrc->p_flag;
        p->p_sflag = psrc->p_sflag;
        p->p_slflag = psrc->p_slflag;
        p->p_lflag = psrc->p_lflag;
        p->p_stflag = psrc->p_stflag;
        p->p_stat = psrc->p_stat;
        p->p_trace_enabled = psrc->p_trace_enabled;
        p->p_pid = psrc->p_pid;
        COND_SET_STRUCT(p->p_pglist, psrc->p_pglist, allowaddr);
        COND_SET_PTR(p->p_pptr, psrc->p_pptr, allowaddr);
        COND_SET_STRUCT(p->p_sibling, psrc->p_sibling, allowaddr);
        COND_SET_STRUCT(p->p_children, psrc->p_children, allowaddr);
        COND_SET_STRUCT(p->p_lwps, psrc->p_lwps, allowaddr);
        COND_SET_PTR(p->p_raslist, psrc->p_raslist, allowaddr);
        p->p_nlwps = psrc->p_nlwps;
        p->p_nzlwps = psrc->p_nzlwps;
        p->p_nrlwps = psrc->p_nrlwps;
        p->p_nlwpwait = psrc->p_nlwpwait;
        p->p_ndlwps = psrc->p_ndlwps;
        p->p_nstopchild = psrc->p_nstopchild;
        p->p_waited = psrc->p_waited;
        COND_SET_PTR(p->p_zomblwp, psrc->p_zomblwp, allowaddr);
        COND_SET_PTR(p->p_vforklwp, psrc->p_vforklwp, allowaddr);
        COND_SET_PTR(p->p_sched_info, psrc->p_sched_info, allowaddr);
        p->p_estcpu = psrc->p_estcpu;
        p->p_estcpu_inherited = psrc->p_estcpu_inherited;
        p->p_forktime = psrc->p_forktime;
        p->p_pctcpu = psrc->p_pctcpu;
        COND_SET_PTR(p->p_opptr, psrc->p_opptr, allowaddr);
        COND_SET_PTR(p->p_timers, psrc->p_timers, allowaddr);
        p->p_rtime = psrc->p_rtime;
        p->p_uticks = psrc->p_uticks;
        p->p_sticks = psrc->p_sticks;
        p->p_iticks = psrc->p_iticks;
        p->p_xutime = psrc->p_xutime;
        p->p_xstime = psrc->p_xstime;
        p->p_traceflag = psrc->p_traceflag;
        COND_SET_PTR(p->p_tracep, psrc->p_tracep, allowaddr);
        COND_SET_PTR(p->p_textvp, psrc->p_textvp, allowaddr);
        COND_SET_PTR(p->p_emul, psrc->p_emul, allowaddr);
        COND_SET_PTR(p->p_emuldata, psrc->p_emuldata, allowaddr);
        COND_SET_CPTR(p->p_execsw, psrc->p_execsw, allowaddr);
        COND_SET_STRUCT(p->p_klist, psrc->p_klist, allowaddr);
        COND_SET_STRUCT(p->p_sigwaiters, psrc->p_sigwaiters, allowaddr);
        COND_SET_STRUCT(p->p_sigpend.sp_info, psrc->p_sigpend.sp_info,
            allowaddr);
        p->p_sigpend.sp_set = psrc->p_sigpend.sp_set;
        COND_SET_PTR(p->p_lwpctl, psrc->p_lwpctl, allowaddr);
        p->p_ppid = psrc->p_ppid;
        p->p_oppid = psrc->p_oppid;
        COND_SET_PTR(p->p_path, psrc->p_path, allowaddr);
        p->p_sigctx = psrc->p_sigctx;
        p->p_nice = psrc->p_nice;
        memcpy(p->p_comm, psrc->p_comm, sizeof(p->p_comm));
        COND_SET_PTR(p->p_pgrp, psrc->p_pgrp, allowaddr);
        COND_SET_VALUE(p->p_psstrp, psrc->p_psstrp, allowaddr);
        p->p_pax = psrc->p_pax;
        p->p_xexit = psrc->p_xexit;
        p->p_xsig = psrc->p_xsig;
        p->p_acflag = psrc->p_acflag;
        COND_SET_STRUCT(p->p_md, psrc->p_md, allowaddr);
        p->p_stackbase = psrc->p_stackbase;
        COND_SET_PTR(p->p_dtrace, psrc->p_dtrace, allowaddr);
}

/*
 * Fill in an eproc structure for the specified process.
 */
void
fill_eproc(struct proc *p, struct eproc *ep, bool zombie, bool allowaddr)
{
        struct tty *tp;
        struct lwp *l;

        KASSERT(mutex_owned(&proc_lock));
        KASSERT(mutex_owned(p->p_lock));

        COND_SET_PTR(ep->e_paddr, p, allowaddr);
        COND_SET_PTR(ep->e_sess, p->p_session, allowaddr);
        if (p->p_cred) {
                kauth_cred_topcred(p->p_cred, &ep->e_pcred);
                kauth_cred_toucred(p->p_cred, &ep->e_ucred);
        }
        if (p->p_stat != SIDL && !P_ZOMBIE(p) && !zombie) {
                struct vmspace *vm = p->p_vmspace;

                ep->e_vm.vm_rssize = vm_resident_count(vm);
                ep->e_vm.vm_tsize = vm->vm_tsize;
                ep->e_vm.vm_dsize = vm->vm_dsize;
                ep->e_vm.vm_ssize = vm->vm_ssize;
                ep->e_vm.vm_map.size = vm->vm_map.size;

                /* Pick the primary (first) LWP */
                l = proc_active_lwp(p);
                KASSERT(l != NULL);
                lwp_lock(l);
                if (l->l_wchan)
                        strncpy(ep->e_wmesg, l->l_wmesg, WMESGLEN);
                lwp_unlock(l);
        }
        ep->e_ppid = p->p_ppid;
        if (p->p_pgrp && p->p_session) {
                ep->e_pgid = p->p_pgrp->pg_id;
                ep->e_jobc = p->p_pgrp->pg_jobc;
                ep->e_sid = p->p_session->s_sid;
                if ((p->p_lflag & PL_CONTROLT) &&
                    (tp = p->p_session->s_ttyp)) {
                        ep->e_tdev = tp->t_dev;
                        ep->e_tpgid = tp->t_pgrp ? tp->t_pgrp->pg_id : NO_PGID;
                        COND_SET_PTR(ep->e_tsess, tp->t_session, allowaddr);
                } else
                        ep->e_tdev = (uint32_t)NODEV;
                ep->e_flag = p->p_session->s_ttyvp ? EPROC_CTTY : 0;
                if (SESS_LEADER(p))
                        ep->e_flag |= EPROC_SLEADER;
                strncpy(ep->e_login, p->p_session->s_login, MAXLOGNAME);
        }
        ep->e_xsize = ep->e_xrssize = 0;
        ep->e_xccount = ep->e_xswrss = 0;
}

/*
 * Fill in a kinfo_proc2 structure for the specified process.
 */
void
fill_kproc2(struct proc *p, struct kinfo_proc2 *ki, bool zombie, bool allowaddr)
{
        struct tty *tp;
        struct lwp *l;
        struct timeval ut, st, rt;
        sigset_t ss1, ss2;
        struct rusage ru;
        struct vmspace *vm;

        KASSERT(mutex_owned(&proc_lock));
        KASSERT(mutex_owned(p->p_lock));

        sigemptyset(&ss1);
        sigemptyset(&ss2);

        COND_SET_VALUE(ki->p_paddr, PTRTOUINT64(p), allowaddr);
        COND_SET_VALUE(ki->p_fd, PTRTOUINT64(p->p_fd), allowaddr);
        COND_SET_VALUE(ki->p_cwdi, PTRTOUINT64(p->p_cwdi), allowaddr);
        COND_SET_VALUE(ki->p_stats, PTRTOUINT64(p->p_stats), allowaddr);
        COND_SET_VALUE(ki->p_limit, PTRTOUINT64(p->p_limit), allowaddr);
        COND_SET_VALUE(ki->p_vmspace, PTRTOUINT64(p->p_vmspace), allowaddr);
        COND_SET_VALUE(ki->p_sigacts, PTRTOUINT64(p->p_sigacts), allowaddr);
        COND_SET_VALUE(ki->p_sess, PTRTOUINT64(p->p_session), allowaddr);
        ki->p_tsess = 0;        /* may be changed if controlling tty below */
        COND_SET_VALUE(ki->p_ru, PTRTOUINT64(&p->p_stats->p_ru), allowaddr);
        ki->p_eflag = 0;
        ki->p_exitsig = p->p_exitsig;
        ki->p_flag = L_INMEM;   /* Process never swapped out */
        ki->p_flag |= sysctl_map_flags(sysctl_flagmap, p->p_flag);
        ki->p_flag |= sysctl_map_flags(sysctl_sflagmap, p->p_sflag);
        ki->p_flag |= sysctl_map_flags(sysctl_slflagmap, p->p_slflag);
        ki->p_flag |= sysctl_map_flags(sysctl_lflagmap, p->p_lflag);
        ki->p_flag |= sysctl_map_flags(sysctl_stflagmap, p->p_stflag);
        ki->p_pid = p->p_pid;
        ki->p_ppid = p->p_ppid;
        ki->p_uid = kauth_cred_geteuid(p->p_cred);
        ki->p_ruid = kauth_cred_getuid(p->p_cred);
        ki->p_gid = kauth_cred_getegid(p->p_cred);
        ki->p_rgid = kauth_cred_getgid(p->p_cred);
        ki->p_svuid = kauth_cred_getsvuid(p->p_cred);
        ki->p_svgid = kauth_cred_getsvgid(p->p_cred);
        ki->p_ngroups = kauth_cred_ngroups(p->p_cred);
        kauth_cred_getgroups(p->p_cred, ki->p_groups,
            uimin(ki->p_ngroups, sizeof(ki->p_groups) / sizeof(ki->p_groups[0])),
            UIO_SYSSPACE);

        ki->p_uticks = p->p_uticks;
        ki->p_sticks = p->p_sticks;
        ki->p_iticks = p->p_iticks;
        ki->p_tpgid = NO_PGID;  /* may be changed if controlling tty below */
        COND_SET_VALUE(ki->p_tracep, PTRTOUINT64(p->p_tracep), allowaddr);
        ki->p_traceflag = p->p_traceflag;

        memcpy(&ki->p_sigignore, &p->p_sigctx.ps_sigignore,sizeof(ki_sigset_t));
        memcpy(&ki->p_sigcatch, &p->p_sigctx.ps_sigcatch, sizeof(ki_sigset_t));

        ki->p_cpticks = 0;
        ki->p_pctcpu = p->p_pctcpu;
        ki->p_estcpu = 0;
        ki->p_stat = p->p_stat; /* Will likely be overridden by LWP status */
        ki->p_realstat = p->p_stat;
        ki->p_nice = p->p_nice;
        ki->p_xstat = P_WAITSTATUS(p);
        ki->p_acflag = p->p_acflag;

        strncpy(ki->p_comm, p->p_comm,
            uimin(sizeof(ki->p_comm), sizeof(p->p_comm)));
        strncpy(ki->p_ename, p->p_emul->e_name, sizeof(ki->p_ename));

        ki->p_nlwps = p->p_nlwps;
        ki->p_realflag = ki->p_flag;

        if (p->p_stat != SIDL && !P_ZOMBIE(p) && !zombie) {
                vm = p->p_vmspace;
                ki->p_vm_rssize = vm_resident_count(vm);
                ki->p_vm_tsize = vm->vm_tsize;
                ki->p_vm_dsize = vm->vm_dsize;
                ki->p_vm_ssize = vm->vm_ssize;
                ki->p_vm_vsize = atop(vm->vm_map.size);
                /*
                 * Since the stack is initially mapped mostly with
                 * PROT_NONE and grown as needed, adjust the "mapped size"
                 * to skip the unused stack portion.
                 */
                ki->p_vm_msize =
                    atop(vm->vm_map.size) - vm->vm_issize + vm->vm_ssize;

                /* Pick the primary (first) LWP */
                l = proc_active_lwp(p);
                KASSERT(l != NULL);
                lwp_lock(l);
                ki->p_nrlwps = p->p_nrlwps;
                ki->p_forw = 0;
                ki->p_back = 0;
                COND_SET_VALUE(ki->p_addr, PTRTOUINT64(l->l_addr), allowaddr);
                ki->p_stat = l->l_stat;
                ki->p_flag |= sysctl_map_flags(sysctl_lwpflagmap, l->l_flag);
                ki->p_swtime = l->l_swtime;
                ki->p_slptime = l->l_slptime;
                if (l->l_stat == LSONPROC)
                        ki->p_schedflags = l->l_cpu->ci_schedstate.spc_flags;
                else
                        ki->p_schedflags = 0;
                ki->p_priority = lwp_eprio(l);
                ki->p_usrpri = l->l_priority;
                if (l->l_wchan)
                        strncpy(ki->p_wmesg, l->l_wmesg, sizeof(ki->p_wmesg));
                COND_SET_VALUE(ki->p_wchan, PTRTOUINT64(l->l_wchan), allowaddr);
                ki->p_cpuid = cpu_index(l->l_cpu);
                lwp_unlock(l);
                LIST_FOREACH(l, &p->p_lwps, l_sibling) {
                        /* This is hardly correct, but... */
                        sigplusset(&l->l_sigpend.sp_set, &ss1);
                        sigplusset(&l->l_sigmask, &ss2);
                        ki->p_cpticks += l->l_cpticks;
                        ki->p_pctcpu += l->l_pctcpu;
                        ki->p_estcpu += l->l_estcpu;
                }
        }
        sigplusset(&p->p_sigpend.sp_set, &ss1);
        memcpy(&ki->p_siglist, &ss1, sizeof(ki_sigset_t));
        memcpy(&ki->p_sigmask, &ss2, sizeof(ki_sigset_t));

        if (p->p_session != NULL) {
                ki->p_sid = p->p_session->s_sid;
                ki->p__pgid = p->p_pgrp->pg_id;
                if (p->p_session->s_ttyvp)
                        ki->p_eflag |= EPROC_CTTY;
                if (SESS_LEADER(p))
                        ki->p_eflag |= EPROC_SLEADER;
                strncpy(ki->p_login, p->p_session->s_login,
                    uimin(sizeof ki->p_login - 1, sizeof p->p_session->s_login));
                ki->p_jobc = p->p_pgrp->pg_jobc;
                if ((p->p_lflag & PL_CONTROLT) && (tp = p->p_session->s_ttyp)) {
                        ki->p_tdev = tp->t_dev;
                        ki->p_tpgid = tp->t_pgrp ? tp->t_pgrp->pg_id : NO_PGID;
                        COND_SET_VALUE(ki->p_tsess, PTRTOUINT64(tp->t_session),
                            allowaddr);
                } else {
                        ki->p_tdev = (int32_t)NODEV;
                }
        }

        if (!P_ZOMBIE(p) && !zombie) {
                ki->p_uvalid = 1;
                ki->p_ustart_sec = p->p_stats->p_start.tv_sec;
                ki->p_ustart_usec = p->p_stats->p_start.tv_usec;

                calcru(p, &ut, &st, NULL, &rt);
                ki->p_rtime_sec = rt.tv_sec;
                ki->p_rtime_usec = rt.tv_usec;
                ki->p_uutime_sec = ut.tv_sec;
                ki->p_uutime_usec = ut.tv_usec;
                ki->p_ustime_sec = st.tv_sec;
                ki->p_ustime_usec = st.tv_usec;

                memcpy(&ru, &p->p_stats->p_ru, sizeof(ru));
                rulwps(p, &ru);
                ki->p_uru_nvcsw = ru.ru_nvcsw;
                ki->p_uru_nivcsw = ru.ru_nivcsw;
                ki->p_uru_maxrss = ru.ru_maxrss;
                ki->p_uru_ixrss = ru.ru_ixrss;
                ki->p_uru_idrss = ru.ru_idrss;
                ki->p_uru_isrss = ru.ru_isrss;
                ki->p_uru_minflt = ru.ru_minflt;
                ki->p_uru_majflt = ru.ru_majflt;
                ki->p_uru_nswap = ru.ru_nswap;
                ki->p_uru_inblock = ru.ru_inblock;
                ki->p_uru_oublock = ru.ru_oublock;
                ki->p_uru_msgsnd = ru.ru_msgsnd;
                ki->p_uru_msgrcv = ru.ru_msgrcv;
                ki->p_uru_nsignals = ru.ru_nsignals;

                timeradd(&p->p_stats->p_cru.ru_utime,
                         &p->p_stats->p_cru.ru_stime, &ut);
                ki->p_uctime_sec = ut.tv_sec;
                ki->p_uctime_usec = ut.tv_usec;
        }
}


int
proc_find_locked(struct lwp *l, struct proc **p, pid_t pid)
{
        int error;

        mutex_enter(&proc_lock);
        if (pid == -1)
                *p = l->l_proc;
        else
                *p = proc_find(pid);

        if (*p == NULL) {
                if (pid != -1)
                        mutex_exit(&proc_lock);
                return SET_ERROR(ESRCH);
        }
        if (pid != -1)
                mutex_enter((*p)->p_lock);
        mutex_exit(&proc_lock);

        error = kauth_authorize_process(l->l_cred,
            KAUTH_PROCESS_CANSEE, *p,
            KAUTH_ARG(KAUTH_REQ_PROCESS_CANSEE_ENTRY), NULL, NULL);
        if (error) {
                if (pid != -1)
                        mutex_exit((*p)->p_lock);
        }
        return error;
}

static int
fill_pathname(struct lwp *l, pid_t pid, void *oldp, size_t *oldlenp)
{
        int error;
        struct proc *p;

        if ((error = proc_find_locked(l, &p, pid)) != 0)
                return error;

        if (p->p_path == NULL) {
                if (pid != -1)
                        mutex_exit(p->p_lock);
                return SET_ERROR(ENOENT);
        }

        size_t len = strlen(p->p_path) + 1;
        if (oldp != NULL) {
                size_t copylen = uimin(len, *oldlenp);
                error = sysctl_copyout(l, p->p_path, oldp, copylen);
                if (error == 0 && *oldlenp < len)
                        error = SET_ERROR(ENOSPC);
        }
        *oldlenp = len;
        if (pid != -1)
                mutex_exit(p->p_lock);
        return error;
}

static int
fill_cwd(struct lwp *l, pid_t pid, void *oldp, size_t *oldlenp)
{
        int error;
        struct proc *p;
        char *path;
        char *bp, *bend;
        struct cwdinfo *cwdi;
        struct vnode *vp;
        size_t len, lenused;

        if ((error = proc_find_locked(l, &p, pid)) != 0)
                return error;

        len = MAXPATHLEN * 4;

        path = kmem_alloc(len, KM_SLEEP);

        bp = &path[len];
        bend = bp;
        *(--bp) = '\0';

        cwdi = p->p_cwdi;
        rw_enter(&cwdi->cwdi_lock, RW_READER);
        vp = cwdi->cwdi_cdir;
        error = getcwd_common(vp, NULL, &bp, path, len/2, 0, l);
        rw_exit(&cwdi->cwdi_lock);

        if (error)
                goto out;

        lenused = bend - bp;

        if (oldp != NULL) {
                size_t copylen = uimin(lenused, *oldlenp);
                error = sysctl_copyout(l, bp, oldp, copylen);
                if (error == 0 && *oldlenp < lenused)
                        error = SET_ERROR(ENOSPC);
        }
        *oldlenp = lenused;
out:
        if (pid != -1)
                mutex_exit(p->p_lock);
        kmem_free(path, len);
        return error;
}

int
proc_getauxv(struct proc *p, void **buf, size_t *len)
{
        struct ps_strings pss;
        int error;
        void *uauxv, *kauxv;
        size_t size;

        if ((error = copyin_psstrings(p, &pss)) != 0)
                return error;
        if (pss.ps_envstr == NULL)
                return SET_ERROR(EIO);

        size = p->p_execsw->es_arglen;
        if (size == 0)
                return SET_ERROR(EIO);

        size_t ptrsz = PROC_PTRSZ(p);
        uauxv = (void *)((char *)pss.ps_envstr + (pss.ps_nenvstr + 1) * ptrsz);

        kauxv = kmem_alloc(size, KM_SLEEP);

        error = copyin_proc(p, uauxv, kauxv, size);
        if (error) {
                kmem_free(kauxv, size);
                return error;
        }

        *buf = kauxv;
        *len = size;

        return 0;
}


static int
sysctl_security_expose_address(SYSCTLFN_ARGS)
{
        int expose_address, error;
        struct sysctlnode node;

        node = *rnode;
        node.sysctl_data = &expose_address;
        expose_address = *(int *)rnode->sysctl_data;
        error = sysctl_lookup(SYSCTLFN_CALL(&node));
        if (error || newp == NULL)
                return error;

        if (kauth_authorize_system(l->l_cred, KAUTH_SYSTEM_KERNADDR,
            0, NULL, NULL, NULL))
                return SET_ERROR(EPERM);

        switch (expose_address) {
        case 0:
        case 1:
        case 2:
                break;
        default:
                return SET_ERROR(EINVAL);
        }

        *(int *)rnode->sysctl_data = expose_address;

        return 0;
}

bool
get_expose_address(struct proc *p)
{
        /* allow only if sysctl variable is set or privileged */
        return kauth_authorize_process(kauth_cred_get(), KAUTH_PROCESS_CANSEE,
            p, KAUTH_ARG(KAUTH_REQ_PROCESS_CANSEE_KPTR), NULL, NULL) == 0;
}