/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
/*
 * Copyright 2017 Joyent, Inc.
 * Copyright 2024 Oxide Computer Company
 */

#include <sys/param.h>
#include <sys/types.h>
#include <sys/stream.h>
#include <sys/strsubr.h>
#include <sys/strsun.h>
#include <sys/stropts.h>
#include <sys/vnode.h>
#include <sys/zone.h>
#include <sys/strlog.h>
#include <sys/sysmacros.h>
#define _SUN_TPI_VERSION 2
#include <sys/tihdr.h>
#include <sys/timod.h>
#include <sys/tiuser.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/sunldi.h>
#include <sys/file.h>
#include <sys/modctl.h>
#include <sys/debug.h>
#include <sys/kmem.h>
#include <sys/cmn_err.h>
#include <sys/proc.h>
#include <sys/suntpi.h>
#include <sys/atomic.h>
#include <sys/mkdev.h>
#include <sys/policy.h>
#include <sys/disp.h>

#include <sys/socket.h>
#include <netinet/in.h>
#include <net/pfkeyv2.h>

#include <inet/common.h>
#include <netinet/ip6.h>
#include <inet/ip.h>
#include <inet/proto_set.h>
#include <inet/nd.h>
#include <inet/optcom.h>
#include <inet/ipsec_info.h>
#include <inet/ipsec_impl.h>
#include <inet/tcp_sig.h>
#include <inet/keysock.h>

#include <sys/isa_defs.h>

/*
 * This is a transport provider for the PF_KEY key management socket.
 * (See RFC 2367 for details.)
 * Downstream messages are wrapped in a keysock consumer interface KEYSOCK_IN
 * messages (see ipsec_info.h), and passed to the appropriate consumer.
 * Upstream messages are generated for all open PF_KEY sockets, when
 * appropriate, as well as the sender (as long as SO_USELOOPBACK is enabled)
 * in reply to downstream messages.
 *
 * Upstream messages must be created asynchronously for the following
 * situations:
 *
 *      1.) A keysock consumer requires an SA, and there is currently none.
 *      2.) An SA expires, either hard or soft lifetime.
 *      3.) Other events a consumer deems fit.
 *
 * The MT model of this is PERMOD, with shared put procedures.  Two types of
 * messages, SADB_FLUSH and SADB_DUMP, need to lock down the perimeter to send
 * down the *multiple* messages they create.
 */

static vmem_t *keysock_vmem;            /* for minor numbers. */

#define KEYSOCK_MAX_CONSUMERS 256

/* Default structure copied into T_INFO_ACK messages (from rts.c...) */
static struct T_info_ack keysock_g_t_info_ack = {
        T_INFO_ACK,
        T_INFINITE,     /* TSDU_size. Maximum size messages. */
        T_INVALID,      /* ETSDU_size. No expedited data. */
        T_INVALID,      /* CDATA_size. No connect data. */
        T_INVALID,      /* DDATA_size. No disconnect data. */
        0,              /* ADDR_size. */
        0,              /* OPT_size. No user-settable options */
        64 * 1024,      /* TIDU_size. keysock allows maximum size messages. */
        T_COTS,         /* SERV_type. keysock supports connection oriented. */
        TS_UNBND,       /* CURRENT_state. This is set from keysock_state. */
        (XPG4_1)        /* Provider flags */
};

/* Named Dispatch Parameter Management Structure */
typedef struct keysockparam_s {
        uint_t  keysock_param_min;
        uint_t  keysock_param_max;
        uint_t  keysock_param_value;
        char    *keysock_param_name;
} keysockparam_t;

/*
 * Table of NDD variables supported by keysock. These are loaded into
 * keysock_g_nd in keysock_init_nd.
 * All of these are alterable, within the min/max values given, at run time.
 */
static  keysockparam_t  lcl_param_arr[] = {
        /* min  max     value   name */
        { 4096, 65536,  8192,   "keysock_xmit_hiwat"},
        { 0,    65536,  1024,   "keysock_xmit_lowat"},
        { 4096, 65536,  8192,   "keysock_recv_hiwat"},
        { 65536, 1024*1024*1024, 256*1024,      "keysock_max_buf"},
        { 0,    3,      0,      "keysock_debug"},
};
#define keystack_xmit_hiwat     keystack_params[0].keysock_param_value
#define keystack_xmit_lowat     keystack_params[1].keysock_param_value
#define keystack_recv_hiwat     keystack_params[2].keysock_param_value
#define keystack_max_buf        keystack_params[3].keysock_param_value
#define keystack_debug  keystack_params[4].keysock_param_value

#define ks0dbg(a)       printf a
/* NOTE:  != 0 instead of > 0 so lint doesn't complain. */
#define ks1dbg(keystack, a)     if (keystack->keystack_debug != 0) printf a
#define ks2dbg(keystack, a)     if (keystack->keystack_debug > 1) printf a
#define ks3dbg(keystack, a)     if (keystack->keystack_debug > 2) printf a

static int keysock_close(queue_t *, int, cred_t *);
static int keysock_open(queue_t *, dev_t *, int, int, cred_t *);
static int keysock_wput(queue_t *, mblk_t *);
static int keysock_rput(queue_t *, mblk_t *);
static int keysock_rsrv(queue_t *);
static void *keysock_stack_init(netstackid_t stackid, netstack_t *ns);
static void keysock_stack_fini(netstackid_t stackid, void *arg);

static struct module_info info = {
        5138, "keysock", 1, INFPSZ, 512, 128
};

static struct qinit rinit = {
        keysock_rput, keysock_rsrv, keysock_open, keysock_close,
        NULL, &info
};

static struct qinit winit = {
        keysock_wput, NULL, NULL, NULL, NULL, &info
};

struct streamtab keysockinfo = {
        &rinit, &winit
};

extern struct modlinkage *keysock_modlp;

/*
 * Plumb IPsec.
 *
 * NOTE:  New "default" modules will need to be loaded here if needed before
 *        boot time.
 */

/* Keep these in global space to keep the lint from complaining. */
static char *IPSECESP = "ipsecesp";
static char *IPSECESPDEV = "/devices/pseudo/ipsecesp@0:ipsecesp";
static char *IPSECAH = "ipsecah";
static char *IPSECAHDEV = "/devices/pseudo/ipsecah@0:ipsecah";
static char *IP6DEV = "/devices/pseudo/ip6@0:ip6";
static char *KEYSOCK = "keysock";
static char *STRMOD = "strmod";

/*
 * Load the other ipsec modules and plumb them together.
 */
int
keysock_plumb_ipsec(netstack_t *ns)
{
        ldi_handle_t    lh, ip6_lh = NULL;
        ldi_ident_t     li = NULL;
        int             err = 0;
        int             muxid, rval;
        boolean_t       esp_present = B_TRUE;
        cred_t          *cr;
        keysock_stack_t *keystack = ns->netstack_keysock;

#ifdef NS_DEBUG
        (void) printf("keysock_plumb_ipsec(%d)\n",
            ns->netstack_stackid);
#endif

        keystack->keystack_plumbed = 0; /* we're trying again.. */

        cr = zone_get_kcred(netstackid_to_zoneid(
            keystack->keystack_netstack->netstack_stackid));
        ASSERT(cr != NULL);
        /*
         * Load up the drivers (AH/ESP).
         *
         * I do this separately from the actual plumbing in case this function
         * ever gets called from a diskless boot before the root filesystem is
         * up.  I don't have to worry about "keysock" because, well, if I'm
         * here, keysock must've loaded successfully.
         */
        if (i_ddi_attach_pseudo_node(IPSECAH) == NULL) {
                ks0dbg(("IPsec:  AH failed to attach.\n"));
                goto bail;
        }
        if (i_ddi_attach_pseudo_node(IPSECESP) == NULL) {
                ks0dbg(("IPsec:  ESP failed to attach.\n"));
                esp_present = B_FALSE;
        }

        /*
         * Set up the IP streams for AH and ESP, as well as tacking keysock
         * on top of them.  Assume keysock has set the autopushes up already.
         */

        /* Open IP. */
        err = ldi_ident_from_mod(keysock_modlp, &li);
        if (err) {
                ks0dbg(("IPsec:  lid_ident_from_mod failed (err %d).\n",
                    err));
                goto bail;
        }

        err = ldi_open_by_name(IP6DEV, FREAD|FWRITE, cr, &ip6_lh, li);
        if (err) {
                ks0dbg(("IPsec:  Open of IP6 failed (err %d).\n", err));
                goto bail;
        }

        /* PLINK KEYSOCK/AH */
        err = ldi_open_by_name(IPSECAHDEV, FREAD|FWRITE, cr, &lh, li);
        if (err) {
                ks0dbg(("IPsec:  Open of AH failed (err %d).\n", err));
                goto bail;
        }
        err = ldi_ioctl(lh,
            I_PUSH, (intptr_t)KEYSOCK, FKIOCTL, cr, &rval);
        if (err) {
                ks0dbg(("IPsec:  Push of KEYSOCK onto AH failed (err %d).\n",
                    err));
                (void) ldi_close(lh, FREAD|FWRITE, cr);
                goto bail;
        }
        err = ldi_ioctl(ip6_lh, I_PLINK, (intptr_t)lh,
            FREAD+FWRITE+FNOCTTY+FKIOCTL, cr, &muxid);
        if (err) {
                ks0dbg(("IPsec:  PLINK of KEYSOCK/AH failed (err %d).\n", err));
                (void) ldi_close(lh, FREAD|FWRITE, cr);
                goto bail;
        }
        (void) ldi_close(lh, FREAD|FWRITE, cr);

        /* PLINK KEYSOCK/ESP */
        if (esp_present) {
                err = ldi_open_by_name(IPSECESPDEV,
                    FREAD|FWRITE, cr, &lh, li);
                if (err) {
                        ks0dbg(("IPsec:  Open of ESP failed (err %d).\n", err));
                        goto bail;
                }
                err = ldi_ioctl(lh,
                    I_PUSH, (intptr_t)KEYSOCK, FKIOCTL, cr, &rval);
                if (err) {
                        ks0dbg(("IPsec:  "
                            "Push of KEYSOCK onto ESP failed (err %d).\n",
                            err));
                        (void) ldi_close(lh, FREAD|FWRITE, cr);
                        goto bail;
                }
                err = ldi_ioctl(ip6_lh, I_PLINK, (intptr_t)lh,
                    FREAD+FWRITE+FNOCTTY+FKIOCTL, cr, &muxid);
                if (err) {
                        ks0dbg(("IPsec:  "
                            "PLINK of KEYSOCK/ESP failed (err %d).\n", err));
                        (void) ldi_close(lh, FREAD|FWRITE, cr);
                        goto bail;
                }
                (void) ldi_close(lh, FREAD|FWRITE, cr);
        }

bail:
        keystack->keystack_plumbed = (err == 0) ? 1 : -1;
        if (ip6_lh != NULL) {
                (void) ldi_close(ip6_lh, FREAD|FWRITE, cr);
        }
        if (li != NULL)
                ldi_ident_release(li);
#ifdef NS_DEBUG
        (void) printf("keysock_plumb_ipsec -> %d\n",
            keystack->keystack_plumbed);
#endif
        crfree(cr);
        return (err);
}

/* ARGSUSED */
static int
keysock_param_get(queue_t *q, mblk_t *mp, caddr_t cp, cred_t *cr)
{
        keysockparam_t  *keysockpa = (keysockparam_t *)cp;
        uint_t value;
        keysock_t *ks = (keysock_t *)q->q_ptr;
        keysock_stack_t *keystack = ks->keysock_keystack;

        mutex_enter(&keystack->keystack_param_lock);
        value = keysockpa->keysock_param_value;
        mutex_exit(&keystack->keystack_param_lock);

        (void) mi_mpprintf(mp, "%u", value);
        return (0);
}

/* This routine sets an NDD variable in a keysockparam_t structure. */
/* ARGSUSED */
static int
keysock_param_set(queue_t *q, mblk_t *mp, char *value, caddr_t cp, cred_t *cr)
{
        ulong_t new_value;
        keysockparam_t  *keysockpa = (keysockparam_t *)cp;
        keysock_t *ks = (keysock_t *)q->q_ptr;
        keysock_stack_t *keystack = ks->keysock_keystack;

        /* Convert the value from a string into a long integer. */
        if (ddi_strtoul(value, NULL, 10, &new_value) != 0)
                return (EINVAL);

        mutex_enter(&keystack->keystack_param_lock);
        /*
         * Fail the request if the new value does not lie within the
         * required bounds.
         */
        if (new_value < keysockpa->keysock_param_min ||
            new_value > keysockpa->keysock_param_max) {
                mutex_exit(&keystack->keystack_param_lock);
                return (EINVAL);
        }

        /* Set the new value */
        keysockpa->keysock_param_value = new_value;
        mutex_exit(&keystack->keystack_param_lock);

        return (0);
}

/*
 * Initialize keysock at module load time
 */
boolean_t
keysock_ddi_init(void)
{
        keysock_max_optsize = optcom_max_optsize(
            keysock_opt_obj.odb_opt_des_arr, keysock_opt_obj.odb_opt_arr_cnt);

        keysock_vmem = vmem_create("keysock", (void *)1, MAXMIN, 1,
            NULL, NULL, NULL, 1, VM_SLEEP | VMC_IDENTIFIER);

        /*
         * We want to be informed each time a stack is created or
         * destroyed in the kernel, so we can maintain the
         * set of keysock_stack_t's.
         */
        netstack_register(NS_KEYSOCK, keysock_stack_init, NULL,
            keysock_stack_fini);

        return (B_TRUE);
}

/*
 * Walk through the param array specified registering each element with the
 * named dispatch handler.
 */
static boolean_t
keysock_param_register(IDP *ndp, keysockparam_t *ksp, int cnt)
{
        for (; cnt-- > 0; ksp++) {
                if (ksp->keysock_param_name != NULL &&
                    ksp->keysock_param_name[0]) {
                        if (!nd_load(ndp,
                            ksp->keysock_param_name,
                            keysock_param_get, keysock_param_set,
                            (caddr_t)ksp)) {
                                nd_free(ndp);
                                return (B_FALSE);
                        }
                }
        }
        return (B_TRUE);
}

/*
 * Initialize keysock for one stack instance
 */
/* ARGSUSED */
static void *
keysock_stack_init(netstackid_t stackid, netstack_t *ns)
{
        keysock_stack_t *keystack;
        keysockparam_t *ksp;

        keystack = (keysock_stack_t *)kmem_zalloc(sizeof (*keystack), KM_SLEEP);
        keystack->keystack_netstack = ns;

        keystack->keystack_acquire_seq = 0xffffffff;

        ksp = (keysockparam_t *)kmem_alloc(sizeof (lcl_param_arr), KM_SLEEP);
        keystack->keystack_params = ksp;
        bcopy(lcl_param_arr, ksp, sizeof (lcl_param_arr));

        (void) keysock_param_register(&keystack->keystack_g_nd, ksp,
            A_CNT(lcl_param_arr));

        mutex_init(&keystack->keystack_list_lock, NULL, MUTEX_DEFAULT, NULL);
        mutex_init(&keystack->keystack_consumers_lock,
            NULL, MUTEX_DEFAULT, NULL);
        mutex_init(&keystack->keystack_param_lock, NULL, MUTEX_DEFAULT, NULL);
        return (keystack);
}

/*
 * Free NDD variable space, and other destructors, for keysock.
 */
void
keysock_ddi_destroy(void)
{
        netstack_unregister(NS_KEYSOCK);
        vmem_destroy(keysock_vmem);
}

/*
 * Remove one stack instance from keysock
 */
/* ARGSUSED */
static void
keysock_stack_fini(netstackid_t stackid, void *arg)
{
        keysock_stack_t *keystack = (keysock_stack_t *)arg;

        nd_free(&keystack->keystack_g_nd);
        kmem_free(keystack->keystack_params, sizeof (lcl_param_arr));
        keystack->keystack_params = NULL;

        mutex_destroy(&keystack->keystack_list_lock);
        mutex_destroy(&keystack->keystack_consumers_lock);
        mutex_destroy(&keystack->keystack_param_lock);

        kmem_free(keystack, sizeof (*keystack));
}

/*
 * Close routine for keysock.
 */
/* ARGSUSED */
static int
keysock_close(queue_t *q, int flags __unused, cred_t *credp __unused)
{
        keysock_t *ks;
        keysock_consumer_t *kc;
        void *ptr = q->q_ptr;
        int size;
        keysock_stack_t *keystack;


        qprocsoff(q);

        /* Safe assumption. */
        ASSERT(ptr != NULL);

        if (WR(q)->q_next) {
                kc = (keysock_consumer_t *)ptr;
                keystack = kc->kc_keystack;

                ks1dbg(keystack, ("Module close, removing a consumer (%d).\n",
                    kc->kc_sa_type));
                /*
                 * Because of PERMOD open/close exclusive perimeter, I
                 * can inspect KC_FLUSHING w/o locking down kc->kc_lock.
                 */
                if (kc->kc_flags & KC_FLUSHING) {
                        /*
                         * If this decrement was the last one, send
                         * down the next pending one, if any.
                         *
                         * With a PERMOD perimeter, the mutexes ops aren't
                         * really necessary, but if we ever loosen up, we will
                         * have this bit covered already.
                         */
                        keystack->keystack_flushdump--;
                        if (keystack->keystack_flushdump == 0) {
                                /*
                                 * The flush/dump terminated by having a
                                 * consumer go away.  I need to send up to the
                                 * appropriate keysock all of the relevant
                                 * information.  Unfortunately, I don't
                                 * have that handy.
                                 */
                                ks0dbg(("Consumer went away while flushing or"
                                    " dumping.\n"));
                        }
                }
                size = sizeof (keysock_consumer_t);
                mutex_enter(&keystack->keystack_consumers_lock);
                keystack->keystack_consumers[kc->kc_sa_type] = NULL;
                mutex_exit(&keystack->keystack_consumers_lock);
                mutex_destroy(&kc->kc_lock);
                netstack_rele(kc->kc_keystack->keystack_netstack);
        } else {
                ks = (keysock_t *)ptr;
                keystack = ks->keysock_keystack;

                ks3dbg(keystack,
                    ("Driver close, PF_KEY socket is going away.\n"));
                if ((ks->keysock_flags & KEYSOCK_EXTENDED) != 0)
                        atomic_dec_32(&keystack->keystack_num_extended);
                size = sizeof (keysock_t);
                mutex_enter(&keystack->keystack_list_lock);
                *(ks->keysock_ptpn) = ks->keysock_next;
                if (ks->keysock_next != NULL)
                        ks->keysock_next->keysock_ptpn = ks->keysock_ptpn;
                mutex_exit(&keystack->keystack_list_lock);
                mutex_destroy(&ks->keysock_lock);
                vmem_free(keysock_vmem, (void *)(uintptr_t)ks->keysock_serial,
                    1);
                netstack_rele(ks->keysock_keystack->keystack_netstack);
        }

        /* Now I'm free. */
        kmem_free(ptr, size);
        return (0);
}
/*
 * Open routine for keysock.
 */
/* ARGSUSED */
static int
keysock_open(queue_t *q, dev_t *devp, int flag, int sflag, cred_t *credp)
{
        keysock_t *ks;
        keysock_consumer_t *kc;
        mblk_t *mp;
        ipsec_info_t *ii;
        netstack_t *ns;
        keysock_stack_t *keystack;

        if (secpolicy_ip_config(credp, B_FALSE) != 0) {
                /* Privilege debugging will log the error */
                return (EPERM);
        }

        if (q->q_ptr != NULL)
                return (0);  /* Re-open of an already open instance. */

        ns = netstack_find_by_cred(credp);
        ASSERT(ns != NULL);
        keystack = ns->netstack_keysock;
        ASSERT(keystack != NULL);

        ks3dbg(keystack, ("Entering keysock open.\n"));

        if (keystack->keystack_plumbed < 1) {
                netstack_t *ns = keystack->keystack_netstack;

                keystack->keystack_plumbed = 0;
#ifdef NS_DEBUG
                printf("keysock_open(%d) - plumb\n",
                    keystack->keystack_netstack->netstack_stackid);
#endif
                /*
                 * Don't worry about ipsec_failure being true here.
                 * (See ip.c).  An open of keysock should try and force
                 * the issue.  Maybe it was a transient failure.
                 */
                ipsec_loader_loadnow(ns->netstack_ipsec);
        }

        if (sflag & MODOPEN) {
                /* Initialize keysock_consumer state here. */
                kc = kmem_zalloc(sizeof (keysock_consumer_t), KM_NOSLEEP);
                if (kc == NULL) {
                        netstack_rele(keystack->keystack_netstack);
                        return (ENOMEM);
                }
                mutex_init(&kc->kc_lock, NULL, MUTEX_DEFAULT, 0);
                kc->kc_rq = q;
                kc->kc_wq = WR(q);

                q->q_ptr = kc;
                WR(q)->q_ptr = kc;

                kc->kc_keystack = keystack;
                qprocson(q);

                /*
                 * Send down initial message to whatever I was pushed on top
                 * of asking for its consumer type.  The reply will set it.
                 */

                /* Allocate it. */
                mp = allocb(sizeof (ipsec_info_t), BPRI_HI);
                if (mp == NULL) {
                        ks1dbg(keystack, (
                            "keysock_open:  Cannot allocate KEYSOCK_HELLO.\n"));
                        /* Do I need to set these to null? */
                        q->q_ptr = NULL;
                        WR(q)->q_ptr = NULL;
                        mutex_destroy(&kc->kc_lock);
                        kmem_free(kc, sizeof (*kc));
                        netstack_rele(keystack->keystack_netstack);
                        return (ENOMEM);
                }

                /* If I allocated okay, putnext to what I was pushed atop. */
                mp->b_wptr += sizeof (ipsec_info_t);
                mp->b_datap->db_type = M_CTL;
                ii = (ipsec_info_t *)mp->b_rptr;
                ii->ipsec_info_type = KEYSOCK_HELLO;
                /* Length only of type/len. */
                ii->ipsec_info_len = sizeof (ii->ipsec_allu);
                ks2dbg(keystack, ("Ready to putnext KEYSOCK_HELLO.\n"));
                putnext(kc->kc_wq, mp);
        } else {
                minor_t ksminor;

                /* Initialize keysock state. */

                ks2dbg(keystack, ("Made it into PF_KEY socket open.\n"));

                ksminor = (minor_t)(uintptr_t)
                    vmem_alloc(keysock_vmem, 1, VM_NOSLEEP);
                if (ksminor == 0) {
                        netstack_rele(keystack->keystack_netstack);
                        return (ENOMEM);
                }
                ks = kmem_zalloc(sizeof (keysock_t), KM_NOSLEEP);
                if (ks == NULL) {
                        vmem_free(keysock_vmem, (void *)(uintptr_t)ksminor, 1);
                        netstack_rele(keystack->keystack_netstack);
                        return (ENOMEM);
                }

                mutex_init(&ks->keysock_lock, NULL, MUTEX_DEFAULT, 0);
                ks->keysock_rq = q;
                ks->keysock_wq = WR(q);
                ks->keysock_state = TS_UNBND;
                ks->keysock_serial = ksminor;

                q->q_ptr = ks;
                WR(q)->q_ptr = ks;
                ks->keysock_keystack = keystack;

                /*
                 * The receive hiwat is only looked at on the stream head
                 * queue.  Store in q_hiwat in order to return on SO_RCVBUF
                 * getsockopts.
                 */

                q->q_hiwat = keystack->keystack_recv_hiwat;

                /*
                 * The transmit hiwat/lowat is only looked at on IP's queue.
                 * Store in q_hiwat/q_lowat in order to return on
                 * SO_SNDBUF/SO_SNDLOWAT getsockopts.
                 */

                WR(q)->q_hiwat = keystack->keystack_xmit_hiwat;
                WR(q)->q_lowat = keystack->keystack_xmit_lowat;

                *devp = makedevice(getmajor(*devp), ksminor);

                /*
                 * Thread keysock into the global keysock list.
                 */
                mutex_enter(&keystack->keystack_list_lock);
                ks->keysock_next = keystack->keystack_list;
                ks->keysock_ptpn = &keystack->keystack_list;
                if (keystack->keystack_list != NULL) {
                        keystack->keystack_list->keysock_ptpn =
                            &ks->keysock_next;
                }
                keystack->keystack_list = ks;
                mutex_exit(&keystack->keystack_list_lock);

                qprocson(q);
                (void) proto_set_rx_hiwat(q, NULL,
                    keystack->keystack_recv_hiwat);
                /*
                 * Wait outside the keysock module perimeter for IPsec
                 * plumbing to be completed.  If it fails, keysock_close()
                 * undoes everything we just did.
                 */
                if (!ipsec_loader_wait(q,
                    keystack->keystack_netstack->netstack_ipsec)) {
                        (void) keysock_close(q, 0, credp);
                        return (EPFNOSUPPORT);
                }
        }

        return (0);
}

/* BELOW THIS LINE ARE ROUTINES INCLUDING AND RELATED TO keysock_wput(). */

/*
 * Copy relevant state bits.
 */
static void
keysock_copy_info(struct T_info_ack *tap, keysock_t *ks)
{
        *tap = keysock_g_t_info_ack;
        tap->CURRENT_state = ks->keysock_state;
        tap->OPT_size = keysock_max_optsize;
}

/*
 * This routine responds to T_CAPABILITY_REQ messages.  It is called by
 * keysock_wput.  Much of the T_CAPABILITY_ACK information is copied from
 * keysock_g_t_info_ack.  The current state of the stream is copied from
 * keysock_state.
 */
static void
keysock_capability_req(queue_t *q, mblk_t *mp)
{
        keysock_t *ks = (keysock_t *)q->q_ptr;
        t_uscalar_t cap_bits1;
        struct T_capability_ack *tcap;

        cap_bits1 = ((struct T_capability_req *)mp->b_rptr)->CAP_bits1;

        mp = tpi_ack_alloc(mp, sizeof (struct T_capability_ack),
            mp->b_datap->db_type, T_CAPABILITY_ACK);
        if (mp == NULL)
                return;

        tcap = (struct T_capability_ack *)mp->b_rptr;
        tcap->CAP_bits1 = 0;

        if (cap_bits1 & TC1_INFO) {
                keysock_copy_info(&tcap->INFO_ack, ks);
                tcap->CAP_bits1 |= TC1_INFO;
        }

        qreply(q, mp);
}

/*
 * This routine responds to T_INFO_REQ messages. It is called by
 * keysock_wput_other.
 * Most of the T_INFO_ACK information is copied from keysock_g_t_info_ack.
 * The current state of the stream is copied from keysock_state.
 */
static void
keysock_info_req(queue_t *q, mblk_t *mp)
{
        mp = tpi_ack_alloc(mp, sizeof (struct T_info_ack), M_PCPROTO,
            T_INFO_ACK);
        if (mp == NULL)
                return;
        keysock_copy_info((struct T_info_ack *)mp->b_rptr,
            (keysock_t *)q->q_ptr);
        qreply(q, mp);
}

/*
 * keysock_err_ack. This routine creates a
 * T_ERROR_ACK message and passes it
 * upstream.
 */
static void
keysock_err_ack(queue_t *q, mblk_t *mp, int t_error, int sys_error)
{
        if ((mp = mi_tpi_err_ack_alloc(mp, t_error, sys_error)) != NULL)
                qreply(q, mp);
}

/*
 * This routine retrieves the current status of socket options.
 * It returns the size of the option retrieved.
 */
/* ARGSUSED */
int
keysock_opt_get(queue_t *q, int level, int name, uchar_t *ptr)
{
        int *i1 = (int *)ptr;
        keysock_t *ks = (keysock_t *)q->q_ptr;

        switch (level) {
        case SOL_SOCKET:
                mutex_enter(&ks->keysock_lock);
                switch (name) {
                case SO_TYPE:
                        *i1 = SOCK_RAW;
                        break;
                case SO_USELOOPBACK:
                        *i1 = (int)(!((ks->keysock_flags & KEYSOCK_NOLOOP) ==
                            KEYSOCK_NOLOOP));
                        break;
                /*
                 * The following two items can be manipulated,
                 * but changing them should do nothing.
                 */
                case SO_SNDBUF:
                        *i1 = (int)q->q_hiwat;
                        break;
                case SO_RCVBUF:
                        *i1 = (int)(RD(q)->q_hiwat);
                        break;
                }
                mutex_exit(&ks->keysock_lock);
                break;
        default:
                return (0);
        }
        return (sizeof (int));
}

/*
 * This routine sets socket options.
 */
/* ARGSUSED */
int
keysock_opt_set(queue_t *q, uint_t mgmt_flags, int level,
    int name, uint_t inlen, uchar_t *invalp, uint_t *outlenp,
    uchar_t *outvalp, void *thisdg_attrs, cred_t *cr)
{
        int *i1 = (int *)invalp, errno = 0;
        keysock_t *ks = (keysock_t *)q->q_ptr;
        keysock_stack_t *keystack = ks->keysock_keystack;

        switch (level) {
        case SOL_SOCKET:
                mutex_enter(&ks->keysock_lock);
                switch (name) {
                case SO_USELOOPBACK:
                        if (!(*i1))
                                ks->keysock_flags |= KEYSOCK_NOLOOP;
                        else ks->keysock_flags &= ~KEYSOCK_NOLOOP;
                        break;
                case SO_SNDBUF:
                        if (*i1 > keystack->keystack_max_buf)
                                errno = ENOBUFS;
                        else q->q_hiwat = *i1;
                        break;
                case SO_RCVBUF:
                        if (*i1 > keystack->keystack_max_buf) {
                                errno = ENOBUFS;
                        } else {
                                RD(q)->q_hiwat = *i1;
                                (void) proto_set_rx_hiwat(RD(q), NULL, *i1);
                        }
                        break;
                default:
                        errno = EINVAL;
                }
                mutex_exit(&ks->keysock_lock);
                break;
        default:
                errno = EINVAL;
        }
        return (errno);
}

/*
 * Handle STREAMS ioctl copyin for getsockname() for both PF_KEY and
 * PF_POLICY.
 */
void
keysock_spdsock_wput_iocdata(queue_t *q, mblk_t *mp, sa_family_t family)
{
        mblk_t *mp1;
        STRUCT_HANDLE(strbuf, sb);
        /* What size of sockaddr do we need? */
        const uint_t addrlen = sizeof (struct sockaddr);

        /* We only handle TI_GET{MY,PEER}NAME (get{sock,peer}name()). */
        switch (((struct iocblk *)mp->b_rptr)->ioc_cmd) {
        case TI_GETMYNAME:
        case TI_GETPEERNAME:
                break;
        default:
                freemsg(mp);
                return;
        }

        switch (mi_copy_state(q, mp, &mp1)) {
        case -1:
                return;
        case MI_COPY_CASE(MI_COPY_IN, 1):
                break;
        case MI_COPY_CASE(MI_COPY_OUT, 1):
                /*
                 * The address has been copied out, so now
                 * copyout the strbuf.
                 */
                mi_copyout(q, mp);
                return;
        case MI_COPY_CASE(MI_COPY_OUT, 2):
                /*
                 * The address and strbuf have been copied out.
                 * We're done, so just acknowledge the original
                 * M_IOCTL.
                 */
                mi_copy_done(q, mp, 0);
                return;
        default:
                /*
                 * Something strange has happened, so acknowledge
                 * the original M_IOCTL with an EPROTO error.
                 */
                mi_copy_done(q, mp, EPROTO);
                return;
        }

        /*
         * Now we have the strbuf structure for TI_GET{MY,PEER}NAME. Next we
         * copyout the requested address and then we'll copyout the strbuf.
         * Regardless of sockname or peername, we just return a sockaddr with
         * sa_family set.
         */
        STRUCT_SET_HANDLE(sb, ((struct iocblk *)mp->b_rptr)->ioc_flag,
            (void *)mp1->b_rptr);

        if (STRUCT_FGET(sb, maxlen) < addrlen) {
                mi_copy_done(q, mp, EINVAL);
                return;
        }

        mp1 = mi_copyout_alloc(q, mp, STRUCT_FGETP(sb, buf), addrlen, B_TRUE);
        if (mp1 == NULL)
                return;

        STRUCT_FSET(sb, len, addrlen);
        ((struct sockaddr *)mp1->b_wptr)->sa_family = family;
        mp1->b_wptr += addrlen;
        mi_copyout(q, mp);
}

/*
 * Handle STREAMS messages.
 */
static void
keysock_wput_other(queue_t *q, mblk_t *mp)
{
        struct iocblk *iocp;
        int error;
        keysock_t *ks = (keysock_t *)q->q_ptr;
        keysock_stack_t *keystack = ks->keysock_keystack;
        cred_t          *cr;

        switch (mp->b_datap->db_type) {
        case M_PROTO:
        case M_PCPROTO:
                if ((mp->b_wptr - mp->b_rptr) < sizeof (long)) {
                        ks3dbg(keystack, (
                            "keysock_wput_other: Not big enough M_PROTO\n"));
                        freemsg(mp);
                        return;
                }
                switch (((union T_primitives *)mp->b_rptr)->type) {
                case T_CAPABILITY_REQ:
                        keysock_capability_req(q, mp);
                        break;
                case T_INFO_REQ:
                        keysock_info_req(q, mp);
                        break;
                case T_SVR4_OPTMGMT_REQ:
                case T_OPTMGMT_REQ:
                        /*
                         * All Solaris components should pass a db_credp
                         * for this TPI message, hence we ASSERT.
                         * But in case there is some other M_PROTO that looks
                         * like a TPI message sent by some other kernel
                         * component, we check and return an error.
                         */
                        cr = msg_getcred(mp, NULL);
                        ASSERT(cr != NULL);
                        if (cr == NULL) {
                                keysock_err_ack(q, mp, TSYSERR, EINVAL);
                                return;
                        }
                        if (((union T_primitives *)mp->b_rptr)->type ==
                            T_SVR4_OPTMGMT_REQ) {
                                svr4_optcom_req(q, mp, cr, &keysock_opt_obj);
                        } else {
                                tpi_optcom_req(q, mp, cr, &keysock_opt_obj);
                        }
                        break;
                case T_DATA_REQ:
                case T_EXDATA_REQ:
                case T_ORDREL_REQ:
                        /* Illegal for keysock. */
                        freemsg(mp);
                        (void) putnextctl1(RD(q), M_ERROR, EPROTO);
                        break;
                default:
                        /* Not supported by keysock. */
                        keysock_err_ack(q, mp, TNOTSUPPORT, 0);
                        break;
                }
                return;
        case M_IOCDATA:
                keysock_spdsock_wput_iocdata(q, mp, PF_KEY);
                return;
        case M_IOCTL:
                iocp = (struct iocblk *)mp->b_rptr;
                error = EINVAL;

                switch (iocp->ioc_cmd) {
                case TI_GETMYNAME:
                case TI_GETPEERNAME:
                        /*
                         * For pfiles(1) observability with getsockname().
                         * See keysock_spdsock_wput_iocdata() for the rest of
                         * this.
                         */
                        mi_copyin(q, mp, NULL,
                            SIZEOF_STRUCT(strbuf, iocp->ioc_flag));
                        return;
                case ND_SET:
                case ND_GET:
                        if (nd_getset(q, keystack->keystack_g_nd, mp)) {
                                qreply(q, mp);
                                return;
                        } else
                                error = ENOENT;
                        /* FALLTHRU */
                default:
                        miocnak(q, mp, 0, error);
                        return;
                }
        case M_FLUSH:
                if (*mp->b_rptr & FLUSHW) {
                        flushq(q, FLUSHALL);
                        *mp->b_rptr &= ~FLUSHW;
                }
                if (*mp->b_rptr & FLUSHR) {
                        qreply(q, mp);
                        return;
                }
                /* Else FALLTHRU */
        }

        /* If fell through, just black-hole the message. */
        freemsg(mp);
}

/*
 * Transmit a PF_KEY error message to the instance either pointed to
 * by ks, the instance with serial number serial, or more, depending.
 *
 * The faulty message (or a reasonable facsimile thereof) is in mp.
 * This function will free mp or recycle it for delivery, thereby causing
 * the stream head to free it.
 */
void
keysock_error(keysock_t *ks, mblk_t *mp, int error, int diagnostic)
{
        sadb_msg_t *samsg = (sadb_msg_t *)mp->b_rptr;
        keysock_stack_t *keystack = ks->keysock_keystack;

        ASSERT(mp->b_datap->db_type == M_DATA);

        if (samsg->sadb_msg_type < SADB_GETSPI ||
            samsg->sadb_msg_type > SADB_MAX)
                samsg->sadb_msg_type = SADB_RESERVED;

        /*
         * Strip out extension headers.
         */
        ASSERT(mp->b_rptr + sizeof (*samsg) <= mp->b_datap->db_lim);
        mp->b_wptr = mp->b_rptr + sizeof (*samsg);
        samsg->sadb_msg_len = SADB_8TO64(sizeof (sadb_msg_t));
        samsg->sadb_msg_errno = (uint8_t)error;
        samsg->sadb_x_msg_diagnostic = (uint16_t)diagnostic;

        keysock_passup(mp, samsg, ks->keysock_serial, NULL, B_FALSE, keystack);
}

/*
 * Pass down a message to a consumer.  Wrap it in KEYSOCK_IN, and copy
 * in the extv if passed in.
 */
static void
keysock_passdown(keysock_t *ks, mblk_t *mp, uint8_t satype, sadb_ext_t *extv[],
    boolean_t flushmsg)
{
        keysock_consumer_t *kc;
        mblk_t *wrapper;
        keysock_in_t *ksi;
        int i;
        keysock_stack_t *keystack = ks->keysock_keystack;

        wrapper = allocb(sizeof (ipsec_info_t), BPRI_HI);
        if (wrapper == NULL) {
                ks3dbg(keystack, ("keysock_passdown: allocb failed.\n"));
                if (extv[SADB_EXT_KEY_ENCRYPT] != NULL) {
                        bzero(extv[SADB_EXT_KEY_ENCRYPT],
                            SADB_64TO8(
                            extv[SADB_EXT_KEY_ENCRYPT]->sadb_ext_len));
                }
                if (extv[SADB_EXT_KEY_AUTH] != NULL) {
                        bzero(extv[SADB_EXT_KEY_AUTH],
                            SADB_64TO8(
                            extv[SADB_EXT_KEY_AUTH]->sadb_ext_len));
                }
                if (extv[SADB_X_EXT_STR_AUTH] != NULL) {
                        bzero(extv[SADB_X_EXT_STR_AUTH],
                            SADB_64TO8(
                            extv[SADB_X_EXT_STR_AUTH]->sadb_ext_len));
                }
                if (flushmsg) {
                        ks0dbg((
                            "keysock: Downwards flush/dump message failed!\n"));
                        /* If this is true, I hold the perimeter. */
                        keystack->keystack_flushdump--;
                }
                freemsg(mp);
                return;
        }

        wrapper->b_datap->db_type = M_CTL;
        ksi = (keysock_in_t *)wrapper->b_rptr;
        ksi->ks_in_type = KEYSOCK_IN;
        ksi->ks_in_len = sizeof (keysock_in_t);
        if (extv[SADB_EXT_ADDRESS_SRC] != NULL)
                ksi->ks_in_srctype = KS_IN_ADDR_UNKNOWN;
        else ksi->ks_in_srctype = KS_IN_ADDR_NOTTHERE;
        if (extv[SADB_EXT_ADDRESS_DST] != NULL)
                ksi->ks_in_dsttype = KS_IN_ADDR_UNKNOWN;
        else ksi->ks_in_dsttype = KS_IN_ADDR_NOTTHERE;
        for (i = 0; i <= SADB_EXT_MAX; i++)
                ksi->ks_in_extv[i] = extv[i];
        ksi->ks_in_serial = ks->keysock_serial;
        wrapper->b_wptr += sizeof (ipsec_info_t);
        wrapper->b_cont = mp;

        /*
         * Find the appropriate consumer where the message is passed down.
         */
        kc = keystack->keystack_consumers[satype];
        if (kc == NULL) {
                freeb(wrapper);
                keysock_error(ks, mp, EINVAL, SADB_X_DIAGNOSTIC_UNKNOWN_SATYPE);
                if (flushmsg) {
                        ks0dbg((
                            "keysock: Downwards flush/dump message failed!\n"));
                        /* If this is true, I hold the perimeter. */
                        keystack->keystack_flushdump--;
                }
                return;
        }

        /*
         * NOTE: There used to be code in here to spin while a flush or
         *       dump finished.  Keysock now assumes that consumers have enough
         *       MT-savviness to deal with that.
         */

        /*
         * Current consumers (AH and ESP) are guaranteed to return a
         * FLUSH or DUMP message back, so when we reach here, we don't
         * have to worry about keysock_flushdumps.
         */

        putnext(kc->kc_wq, wrapper);
}

/*
 * High-level reality checking of extensions.
 */
static boolean_t
ext_check(sadb_ext_t *ext, keysock_stack_t *keystack)
{
        int i;
        uint64_t *lp;
        sadb_ident_t *id;
        char *idstr;

        switch (ext->sadb_ext_type) {
        case SADB_EXT_ADDRESS_SRC:
        case SADB_EXT_ADDRESS_DST:
        case SADB_X_EXT_ADDRESS_INNER_SRC:
        case SADB_X_EXT_ADDRESS_INNER_DST:
                /* Check for at least enough addtl length for a sockaddr. */
                if (ext->sadb_ext_len <= SADB_8TO64(sizeof (sadb_address_t)))
                        return (B_FALSE);
                break;
        case SADB_EXT_LIFETIME_HARD:
        case SADB_EXT_LIFETIME_SOFT:
        case SADB_EXT_LIFETIME_CURRENT:
                if (ext->sadb_ext_len != SADB_8TO64(sizeof (sadb_lifetime_t)))
                        return (B_FALSE);
                break;
        case SADB_EXT_SPIRANGE:
                /* See if the SPI range is legit. */
                if (htonl(((sadb_spirange_t *)ext)->sadb_spirange_min) >
                    htonl(((sadb_spirange_t *)ext)->sadb_spirange_max))
                        return (B_FALSE);
                break;
        case SADB_EXT_KEY_AUTH:
        case SADB_EXT_KEY_ENCRYPT:
                /* Key length check. */
                if (((sadb_key_t *)ext)->sadb_key_bits == 0)
                        return (B_FALSE);
                /*
                 * Check to see if the key length (in bits) is less than the
                 * extension length (in 8-bits words).
                 */
                if ((roundup(SADB_1TO8(((sadb_key_t *)ext)->sadb_key_bits), 8) +
                    sizeof (sadb_key_t)) != SADB_64TO8(ext->sadb_ext_len)) {
                        ks1dbg(keystack, (
                            "ext_check:  Key bits/length inconsistent.\n"));
                        ks1dbg(keystack, ("%d bits, len is %d bytes.\n",
                            ((sadb_key_t *)ext)->sadb_key_bits,
                            SADB_64TO8(ext->sadb_ext_len)));
                        return (B_FALSE);
                }

                /* All-zeroes key check. */
                lp = (uint64_t *)(((char *)ext) + sizeof (sadb_key_t));
                for (i = 0;
                    i < (ext->sadb_ext_len - SADB_8TO64(sizeof (sadb_key_t)));
                    i++)
                        if (lp[i] != 0)
                                break;  /* Out of for loop. */
                /* If finished the loop naturally, it's an all zero key. */
                if (lp[i] == 0)
                        return (B_FALSE);
                break;
        case SADB_X_EXT_STR_AUTH: {
                sadb_key_t *key = (sadb_key_t *)ext;

                if (key->sadb_key_bits == 0)
                        return (B_FALSE);
                if (key->sadb_key_bits > SADB_8TO1(TCPSIG_MD5_KEY_LEN))
                        return (B_FALSE);
                break;
        }
        case SADB_EXT_IDENTITY_SRC:
        case SADB_EXT_IDENTITY_DST:
                /*
                 * Make sure the strings in these identities are
                 * null-terminated.  RFC 2367 underspecified how to handle
                 * such a case.  I "proactively" null-terminate the string
                 * at the last byte if it's not terminated sooner.
                 */
                id = (sadb_ident_t *)ext;
                i = SADB_64TO8(id->sadb_ident_len);
                i -= sizeof (sadb_ident_t);
                idstr = (char *)(id + 1);
                while (*idstr != '\0' && i > 0) {
                        i--;
                        idstr++;
                }
                if (i == 0) {
                        /*
                         * I.e., if the bozo user didn't NULL-terminate the
                         * string...
                         */
                        idstr--;
                        *idstr = '\0';
                }
                break;
        }
        return (B_TRUE);        /* For now... */
}

/* Return values for keysock_get_ext(). */
#define KGE_OK  0
#define KGE_DUP 1
#define KGE_UNK 2
#define KGE_LEN 3
#define KGE_CHK 4

/*
 * Parse basic extension headers and return in the passed-in pointer vector.
 * Return values include:
 *
 *      KGE_OK  Everything's nice and parsed out.
 *              If there are no extensions, place NULL in extv[0].
 *      KGE_DUP There is a duplicate extension.
 *              First instance in appropriate bin.  First duplicate in
 *              extv[0].
 *      KGE_UNK Unknown extension type encountered.  extv[0] contains
 *              unknown header.
 *      KGE_LEN Extension length error.
 *      KGE_CHK High-level reality check failed on specific extension.
 *
 * My apologies for some of the pointer arithmetic in here.  I'm thinking
 * like an assembly programmer, yet trying to make the compiler happy.
 */
static int
keysock_get_ext(sadb_ext_t *extv[], sadb_msg_t *basehdr, uint_t msgsize,
    keysock_stack_t *keystack)
{
        bzero(extv, sizeof (sadb_ext_t *) * (SADB_EXT_MAX + 1));

        /* Use extv[0] as the "current working pointer". */

        extv[0] = (sadb_ext_t *)(basehdr + 1);

        while (extv[0] < (sadb_ext_t *)(((uint8_t *)basehdr) + msgsize)) {
                /* Check for unknown headers. */
                if (extv[0]->sadb_ext_type == 0 ||
                    extv[0]->sadb_ext_type > SADB_EXT_MAX)
                        return (KGE_UNK);

                /*
                 * Check length.  Use uint64_t because extlen is in units
                 * of 64-bit words.  If length goes beyond the msgsize,
                 * return an error.  (Zero length also qualifies here.)
                 */
                if (extv[0]->sadb_ext_len == 0 ||
                    (void *)((uint64_t *)extv[0] + extv[0]->sadb_ext_len) >
                    (void *)((uint8_t *)basehdr + msgsize))
                        return (KGE_LEN);

                /* Check for redundant headers. */
                if (extv[extv[0]->sadb_ext_type] != NULL)
                        return (KGE_DUP);

                /*
                 * Reality check the extension if possible at the keysock
                 * level.
                 */
                if (!ext_check(extv[0], keystack))
                        return (KGE_CHK);

                /* If I make it here, assign the appropriate bin. */
                extv[extv[0]->sadb_ext_type] = extv[0];

                /* Advance pointer (See above for uint64_t ptr reasoning.) */
                extv[0] = (sadb_ext_t *)
                    ((uint64_t *)extv[0] + extv[0]->sadb_ext_len);
        }

        /* Everything's cool. */

        /*
         * If extv[0] == NULL, then there are no extension headers in this
         * message.  Ensure that this is the case.
         */
        if (extv[0] == (sadb_ext_t *)(basehdr + 1))
                extv[0] = NULL;

        return (KGE_OK);
}

/*
 * qwriter() callback to handle flushes and dumps.  This routine will hold
 * the inner perimeter.
 */
void
keysock_do_flushdump(queue_t *q, mblk_t *mp)
{
        int i, start, finish;
        mblk_t *mp1 = NULL;
        keysock_t *ks = (keysock_t *)q->q_ptr;
        sadb_ext_t *extv[SADB_EXT_MAX + 1];
        sadb_msg_t *samsg = (sadb_msg_t *)mp->b_rptr;
        keysock_stack_t *keystack = ks->keysock_keystack;

        /*
         * I am guaranteed this will work.  I did the work in keysock_parse()
         * already.
         */
        (void) keysock_get_ext(extv, samsg, SADB_64TO8(samsg->sadb_msg_len),
            keystack);

        /*
         * I hold the perimeter, therefore I don't need to use atomic ops.
         */
        if (keystack->keystack_flushdump != 0) {
                /* XXX Should I instead use EBUSY? */
                /* XXX Or is there a way to queue these up? */
                keysock_error(ks, mp, ENOMEM, SADB_X_DIAGNOSTIC_NONE);
                return;
        }

        if (samsg->sadb_msg_satype == SADB_SATYPE_UNSPEC) {
                start = 0;
                finish = KEYSOCK_MAX_CONSUMERS - 1;
        } else {
                start = samsg->sadb_msg_satype;
                finish = samsg->sadb_msg_satype;
        }

        /*
         * Fill up keysock_flushdump with the number of outstanding dumps
         * and/or flushes.
         */

        keystack->keystack_flushdump_errno = 0;

        /*
         * Okay, I hold the perimeter.  Eventually keysock_flushdump will
         * contain the number of consumers with outstanding flush operations.
         *
         * SO, here's the plan:
         *      * For each relevant consumer (Might be one, might be all)
         *              * Twiddle on the FLUSHING flag.
         *              * Pass down the FLUSH/DUMP message.
         *
         * When I see upbound FLUSH/DUMP messages, I will decrement the
         * keysock_flushdump.  When I decrement it to 0, I will pass the
         * FLUSH/DUMP message back up to the PF_KEY sockets.  Because I will
         * pass down the right SA type to the consumer (either its own, or
         * that of UNSPEC), the right one will be reflected from each consumer,
         * and accordingly back to the socket.
         */

        mutex_enter(&keystack->keystack_consumers_lock);
        for (i = start; i <= finish; i++) {
                if (keystack->keystack_consumers[i] != NULL) {
                        mp1 = copymsg(mp);
                        if (mp1 == NULL) {
                                ks0dbg(("SADB_FLUSH copymsg() failed.\n"));
                                /*
                                 * Error?  And what about outstanding
                                 * flushes?  Oh, yeah, they get sucked up and
                                 * the counter is decremented.  Consumers
                                 * (see keysock_passdown()) are guaranteed
                                 * to deliver back a flush request, even if
                                 * it's an error.
                                 */
                                keysock_error(ks, mp, ENOMEM,
                                    SADB_X_DIAGNOSTIC_NONE);
                                return;
                        }
                        /*
                         * Because my entry conditions are met above, the
                         * following assertion should hold true.
                         */
                        mutex_enter(&keystack->keystack_consumers[i]->kc_lock);
                        ASSERT((keystack->keystack_consumers[i]->kc_flags &
                            KC_FLUSHING) == 0);
                        keystack->keystack_consumers[i]->kc_flags |=
                            KC_FLUSHING;
                        mutex_exit(&(keystack->keystack_consumers[i]->kc_lock));
                        /* Always increment the number of flushes... */
                        keystack->keystack_flushdump++;
                        /* Guaranteed to return a message. */
                        keysock_passdown(ks, mp1, i, extv, B_TRUE);
                } else if (start == finish) {
                        /*
                         * In case where start == finish, and there's no
                         * consumer, should we force an error?  Yes.
                         */
                        mutex_exit(&keystack->keystack_consumers_lock);
                        keysock_error(ks, mp, EINVAL,
                            SADB_X_DIAGNOSTIC_UNKNOWN_SATYPE);
                        return;
                }
        }
        mutex_exit(&keystack->keystack_consumers_lock);

        if (keystack->keystack_flushdump == 0) {
                /*
                 * There were no consumers at all for this message.
                 * XXX For now return ESRCH.
                 */
                keysock_error(ks, mp, ESRCH, SADB_X_DIAGNOSTIC_NO_SADBS);
        } else {
                /* Otherwise, free the original message. */
                freemsg(mp);
        }
}

/*
 * Get the right diagnostic for a duplicate.  Should probably use a static
 * table lookup.
 */
int
keysock_duplicate(int ext_type)
{
        int rc = 0;

        switch (ext_type) {
        case SADB_EXT_ADDRESS_SRC:
                rc = SADB_X_DIAGNOSTIC_DUPLICATE_SRC;
                break;
        case SADB_EXT_ADDRESS_DST:
                rc = SADB_X_DIAGNOSTIC_DUPLICATE_DST;
                break;
        case SADB_X_EXT_ADDRESS_INNER_SRC:
                rc = SADB_X_DIAGNOSTIC_DUPLICATE_INNER_SRC;
                break;
        case SADB_X_EXT_ADDRESS_INNER_DST:
                rc = SADB_X_DIAGNOSTIC_DUPLICATE_INNER_DST;
                break;
        case SADB_EXT_SA:
                rc = SADB_X_DIAGNOSTIC_DUPLICATE_SA;
                break;
        case SADB_EXT_SPIRANGE:
                rc = SADB_X_DIAGNOSTIC_DUPLICATE_RANGE;
                break;
        case SADB_EXT_KEY_AUTH:
                rc = SADB_X_DIAGNOSTIC_DUPLICATE_AKEY;
                break;
        case SADB_EXT_KEY_ENCRYPT:
                rc = SADB_X_DIAGNOSTIC_DUPLICATE_EKEY;
                break;
        case SADB_X_EXT_STR_AUTH:
                rc = SADB_X_DIAGNOSTIC_DUPLICATE_ASTR;
                break;
        }
        return (rc);
}

/*
 * Get the right diagnostic for a reality check failure.  Should probably use
 * a static table lookup.
 */
int
keysock_malformed(int ext_type)
{
        int rc = 0;

        switch (ext_type) {
        case SADB_EXT_ADDRESS_SRC:
                rc = SADB_X_DIAGNOSTIC_MALFORMED_SRC;
                break;
        case SADB_EXT_ADDRESS_DST:
                rc = SADB_X_DIAGNOSTIC_MALFORMED_DST;
                break;
        case SADB_X_EXT_ADDRESS_INNER_SRC:
                rc = SADB_X_DIAGNOSTIC_MALFORMED_INNER_SRC;
                break;
        case SADB_X_EXT_ADDRESS_INNER_DST:
                rc = SADB_X_DIAGNOSTIC_MALFORMED_INNER_DST;
                break;
        case SADB_EXT_SA:
                rc = SADB_X_DIAGNOSTIC_MALFORMED_SA;
                break;
        case SADB_EXT_SPIRANGE:
                rc = SADB_X_DIAGNOSTIC_MALFORMED_RANGE;
                break;
        case SADB_EXT_KEY_AUTH:
                rc = SADB_X_DIAGNOSTIC_MALFORMED_AKEY;
                break;
        case SADB_EXT_KEY_ENCRYPT:
                rc = SADB_X_DIAGNOSTIC_MALFORMED_EKEY;
                break;
        case SADB_X_EXT_STR_AUTH:
                rc = SADB_X_DIAGNOSTIC_MALFORMED_ASTR;
                break;
        }
        return (rc);
}

/*
 * Keysock massaging of an inverse ACQUIRE.  Consult policy,
 * and construct an appropriate response.
 */
static void
keysock_inverse_acquire(mblk_t *mp, sadb_msg_t *samsg, sadb_ext_t *extv[],
    keysock_t *ks)
{
        mblk_t *reply_mp;
        keysock_stack_t *keystack = ks->keysock_keystack;

        /*
         * Reality check things...
         */
        if (extv[SADB_EXT_ADDRESS_SRC] == NULL) {
                keysock_error(ks, mp, EINVAL, SADB_X_DIAGNOSTIC_MISSING_SRC);
                return;
        }
        if (extv[SADB_EXT_ADDRESS_DST] == NULL) {
                keysock_error(ks, mp, EINVAL, SADB_X_DIAGNOSTIC_MISSING_DST);
                return;
        }

        if (extv[SADB_X_EXT_ADDRESS_INNER_SRC] != NULL &&
            extv[SADB_X_EXT_ADDRESS_INNER_DST] == NULL) {
                keysock_error(ks, mp, EINVAL,
                    SADB_X_DIAGNOSTIC_MISSING_INNER_DST);
                return;
        }

        if (extv[SADB_X_EXT_ADDRESS_INNER_SRC] == NULL &&
            extv[SADB_X_EXT_ADDRESS_INNER_DST] != NULL) {
                keysock_error(ks, mp, EINVAL,
                    SADB_X_DIAGNOSTIC_MISSING_INNER_SRC);
                return;
        }

        reply_mp = ipsec_construct_inverse_acquire(samsg, extv,
            keystack->keystack_netstack);

        if (reply_mp != NULL) {
                freemsg(mp);
                keysock_passup(reply_mp, (sadb_msg_t *)reply_mp->b_rptr,
                    ks->keysock_serial, NULL, B_FALSE, keystack);
        } else {
                keysock_error(ks, mp, samsg->sadb_msg_errno,
                    samsg->sadb_x_msg_diagnostic);
        }
}

/*
 * Spew an extended REGISTER down to the relevant consumers.
 */
static void
keysock_extended_register(keysock_t *ks, mblk_t *mp, sadb_ext_t *extv[])
{
        sadb_x_ereg_t *ereg = (sadb_x_ereg_t *)extv[SADB_X_EXT_EREG];
        uint8_t *satypes, *fencepost;
        mblk_t *downmp;
        sadb_ext_t *downextv[SADB_EXT_MAX + 1];
        keysock_stack_t *keystack = ks->keysock_keystack;

        if (ks->keysock_registered[0] != 0 || ks->keysock_registered[1] != 0 ||
            ks->keysock_registered[2] != 0 || ks->keysock_registered[3] != 0) {
                keysock_error(ks, mp, EBUSY, 0);
        }

        ks->keysock_flags |= KEYSOCK_EXTENDED;
        if (ereg == NULL) {
                keysock_error(ks, mp, EINVAL, SADB_X_DIAGNOSTIC_SATYPE_NEEDED);
        } else {
                ASSERT(mp->b_rptr + msgdsize(mp) == mp->b_wptr);
                fencepost = (uint8_t *)mp->b_wptr;
                satypes = ereg->sadb_x_ereg_satypes;
                while (*satypes != SADB_SATYPE_UNSPEC && satypes != fencepost) {
                        downmp = copymsg(mp);
                        if (downmp == NULL) {
                                keysock_error(ks, mp, ENOMEM, 0);
                                return;
                        }
                        /*
                         * Since we've made it here, keysock_get_ext will work!
                         */
                        (void) keysock_get_ext(downextv,
                            (sadb_msg_t *)downmp->b_rptr, msgdsize(downmp),
                            keystack);
                        keysock_passdown(ks, downmp, *satypes, downextv,
                            B_FALSE);
                        ++satypes;
                }
                freemsg(mp);
        }

        /*
         * Set global to indicate we prefer an extended ACQUIRE.
         */
        atomic_inc_32(&keystack->keystack_num_extended);
}

static void
keysock_delpair_all(keysock_t *ks, mblk_t *mp, sadb_ext_t *extv[])
{
        int i, start, finish;
        mblk_t *mp1 = NULL;
        keysock_stack_t *keystack = ks->keysock_keystack;

        start = 0;
        finish = KEYSOCK_MAX_CONSUMERS - 1;

        for (i = start; i <= finish; i++) {
                if (keystack->keystack_consumers[i] != NULL) {
                        mp1 = copymsg(mp);
                        if (mp1 == NULL) {
                                keysock_error(ks, mp, ENOMEM,
                                    SADB_X_DIAGNOSTIC_NONE);
                                return;
                        }
                        keysock_passdown(ks, mp1, i, extv, B_FALSE);
                }
        }
}

/*
 * Handle PF_KEY messages.
 */
static void
keysock_parse(queue_t *q, mblk_t *mp)
{
        sadb_msg_t *samsg;
        sadb_ext_t *extv[SADB_EXT_MAX + 1];
        keysock_t *ks = (keysock_t *)q->q_ptr;
        uint_t msgsize;
        uint8_t satype;
        keysock_stack_t *keystack = ks->keysock_keystack;

        /* Make sure I'm a PF_KEY socket.  (i.e. nothing's below me) */
        ASSERT(WR(q)->q_next == NULL);

        samsg = (sadb_msg_t *)mp->b_rptr;
        ks2dbg(keystack, ("Received possible PF_KEY message, type %d.\n",
            samsg->sadb_msg_type));

        msgsize = SADB_64TO8(samsg->sadb_msg_len);

        if (msgdsize(mp) != msgsize) {
                /*
                 * Message len incorrect w.r.t. actual size.  Send an error
                 * (EMSGSIZE).  It may be necessary to massage things a
                 * bit.  For example, if the sadb_msg_type is hosed,
                 * I need to set it to SADB_RESERVED to get delivery to
                 * do the right thing.  Then again, maybe just letting
                 * the error delivery do the right thing.
                 */
                ks2dbg(keystack,
                    ("mblk (%lu) and base (%d) message sizes don't jibe.\n",
                    msgdsize(mp), msgsize));
                keysock_error(ks, mp, EMSGSIZE, SADB_X_DIAGNOSTIC_NONE);
                return;
        }

        if (msgsize > (uint_t)(mp->b_wptr - mp->b_rptr)) {
                /* Get all message into one mblk. */
                if (pullupmsg(mp, -1) == 0) {
                        /*
                         * Something screwy happened.
                         */
                        ks3dbg(keystack,
                            ("keysock_parse: pullupmsg() failed.\n"));
                        return;
                } else {
                        samsg = (sadb_msg_t *)mp->b_rptr;
                }
        }

        switch (keysock_get_ext(extv, samsg, msgsize, keystack)) {
        case KGE_DUP:
                /* Handle duplicate extension. */
                ks1dbg(keystack, ("Got duplicate extension of type %d.\n",
                    extv[0]->sadb_ext_type));
                keysock_error(ks, mp, EINVAL,
                    keysock_duplicate(extv[0]->sadb_ext_type));
                return;
        case KGE_UNK:
                /* Handle unknown extension. */
                ks1dbg(keystack, ("Got unknown extension of type %d.\n",
                    extv[0]->sadb_ext_type));
                keysock_error(ks, mp, EINVAL, SADB_X_DIAGNOSTIC_UNKNOWN_EXT);
                return;
        case KGE_LEN:
                /* Length error. */
                ks1dbg(keystack,
                    ("Length %d on extension type %d overrun or 0.\n",
                    extv[0]->sadb_ext_len, extv[0]->sadb_ext_type));
                keysock_error(ks, mp, EINVAL, SADB_X_DIAGNOSTIC_BAD_EXTLEN);
                return;
        case KGE_CHK:
                /* Reality check failed. */
                ks1dbg(keystack,
                    ("Reality check failed on extension type %d.\n",
                    extv[0]->sadb_ext_type));
                keysock_error(ks, mp, EINVAL,
                    keysock_malformed(extv[0]->sadb_ext_type));
                return;
        default:
                /* Default case is no errors. */
                break;
        }

        /*
         * If this is a TCPSIG SA message, pass it off to the handler in
         * tcp_sig.c and return. This is not implemented as a downstream
         * module.
         */
        if (samsg->sadb_msg_satype == SADB_X_SATYPE_TCPSIG) {
                tcpsig_sa_handler(ks, mp, samsg, extv);
                return;
        }

        switch (samsg->sadb_msg_type) {
        case SADB_REGISTER:
                /*
                 * There's a semantic weirdness in that a message OTHER than
                 * the return REGISTER message may be passed up if I set the
                 * registered bit BEFORE I pass it down.
                 *
                 * SOOOO, I'll not twiddle any registered bits until I see
                 * the upbound REGISTER (with a serial number in it).
                 */
                if (samsg->sadb_msg_satype == SADB_SATYPE_UNSPEC) {
                        /* Handle extended register here. */
                        keysock_extended_register(ks, mp, extv);
                        return;
                } else if (ks->keysock_flags & KEYSOCK_EXTENDED) {
                        keysock_error(ks, mp, EBUSY, 0);
                        return;
                }
                /* FALLTHRU */
        case SADB_GETSPI:
        case SADB_ADD:
        case SADB_UPDATE:
        case SADB_X_UPDATEPAIR:
        case SADB_DELETE:
        case SADB_X_DELPAIR:
        case SADB_GET:
                /*
                 * Pass down to appropriate consumer.
                 */
                if (samsg->sadb_msg_satype != SADB_SATYPE_UNSPEC) {
                        keysock_passdown(ks, mp, samsg->sadb_msg_satype, extv,
                            B_FALSE);
                } else {
                        keysock_error(ks, mp, EINVAL,
                            SADB_X_DIAGNOSTIC_SATYPE_NEEDED);
                }
                return;
        case SADB_X_DELPAIR_STATE:
                if (samsg->sadb_msg_satype == SADB_SATYPE_UNSPEC) {
                        keysock_delpair_all(ks, mp, extv);
                } else {
                        keysock_passdown(ks, mp, samsg->sadb_msg_satype, extv,
                            B_FALSE);
                }
                return;
        case SADB_ACQUIRE:
                /*
                 * If I _receive_ an acquire, this means I should spread it
                 * out to registered sockets.  Unless there's an errno...
                 *
                 * Need ADDRESS, may have ID, SENS, and PROP, unless errno,
                 * in which case there should be NO extensions.
                 *
                 * Return to registered.
                 */
                if (samsg->sadb_msg_errno != 0) {
                        satype = samsg->sadb_msg_satype;
                        if (satype == SADB_SATYPE_UNSPEC) {
                                if (!(ks->keysock_flags & KEYSOCK_EXTENDED)) {
                                        keysock_error(ks, mp, EINVAL,
                                            SADB_X_DIAGNOSTIC_SATYPE_NEEDED);
                                        return;
                                }
                                /*
                                 * Reassign satype based on the first
                                 * flags that KEYSOCK_SETREG says.
                                 */
                                while (satype <= SADB_SATYPE_MAX) {
                                        if (KEYSOCK_ISREG(ks, satype))
                                                break;
                                        satype++;
                                }
                                if (satype > SADB_SATYPE_MAX) {
                                        keysock_error(ks, mp, EBUSY, 0);
                                        return;
                                }
                        }
                        keysock_passdown(ks, mp, satype, extv, B_FALSE);
                } else {
                        if (samsg->sadb_msg_satype == SADB_SATYPE_UNSPEC) {
                                keysock_error(ks, mp, EINVAL,
                                    SADB_X_DIAGNOSTIC_SATYPE_NEEDED);
                        } else {
                                keysock_passup(mp, samsg, 0, NULL, B_FALSE,
                                    keystack);
                        }
                }
                return;
        case SADB_EXPIRE:
                /*
                 * If someone sends this in, then send out to all senders.
                 * (Save maybe ESP or AH, I have to be careful here.)
                 *
                 * Need ADDRESS, may have ID and SENS.
                 *
                 * XXX for now this is unsupported.
                 */
                break;
        case SADB_FLUSH:
                /*
                 * Nuke all SAs.
                 *
                 * No extensions at all.  Return to all listeners.
                 *
                 * Question:    Should I hold a lock here to prevent
                 *              additions/deletions while flushing?
                 * Answer:      No.  (See keysock_passdown() for details.)
                 */
                if (extv[0] != NULL) {
                        /*
                         * FLUSH messages shouldn't have extensions.
                         * Return EINVAL.
                         */
                        ks2dbg(keystack, ("FLUSH message with extension.\n"));
                        keysock_error(ks, mp, EINVAL, SADB_X_DIAGNOSTIC_NO_EXT);
                        return;
                }

                /* Passing down of DUMP/FLUSH messages are special. */
                qwriter(q, mp, keysock_do_flushdump, PERIM_INNER);
                return;
        case SADB_DUMP:  /* not used by normal applications */
                if ((extv[0] != NULL) &&
                    ((msgsize >
                    (sizeof (sadb_msg_t) + sizeof (sadb_x_edump_t))) ||
                    (extv[SADB_X_EXT_EDUMP] == NULL))) {
                                keysock_error(ks, mp, EINVAL,
                                    SADB_X_DIAGNOSTIC_NO_EXT);
                                return;
                }
                qwriter(q, mp, keysock_do_flushdump, PERIM_INNER);
                return;
        case SADB_X_PROMISC:
                /*
                 * Promiscuous processing message.
                 */
                if (samsg->sadb_msg_satype == 0)
                        ks->keysock_flags &= ~KEYSOCK_PROMISC;
                else
                        ks->keysock_flags |= KEYSOCK_PROMISC;
                keysock_passup(mp, samsg, ks->keysock_serial, NULL, B_FALSE,
                    keystack);
                return;
        case SADB_X_INVERSE_ACQUIRE:
                keysock_inverse_acquire(mp, samsg, extv, ks);
                return;
        default:
                ks2dbg(keystack, ("Got unknown message type %d.\n",
                    samsg->sadb_msg_type));
                keysock_error(ks, mp, EINVAL, SADB_X_DIAGNOSTIC_UNKNOWN_MSG);
                return;
        }

        /* As a placeholder... */
        ks0dbg(("keysock_parse():  Hit EOPNOTSUPP\n"));
        keysock_error(ks, mp, EOPNOTSUPP, SADB_X_DIAGNOSTIC_NONE);
}

/*
 * wput routing for PF_KEY/keysock/whatever.  Unlike the routing socket,
 * I don't convert to ioctl()'s for IP.  I am the end-all driver as far
 * as PF_KEY sockets are concerned.  I do some conversion, but not as much
 * as IP/rts does.
 */
static int
keysock_wput(queue_t *q, mblk_t *mp)
{
        uchar_t *rptr = mp->b_rptr;
        mblk_t *mp1;
        keysock_t *ks;
        keysock_stack_t *keystack;

        if (WR(q)->q_next) {
                keysock_consumer_t *kc = (keysock_consumer_t *)q->q_ptr;
                keystack = kc->kc_keystack;

                ks3dbg(keystack, ("In keysock_wput\n"));

                /*
                 * We shouldn't get writes on a consumer instance.
                 * But for now, just passthru.
                 */
                ks1dbg(keystack, ("Huh?  wput for an consumer instance (%d)?\n",
                    kc->kc_sa_type));
                putnext(q, mp);
                return (0);
        }
        ks = (keysock_t *)q->q_ptr;
        keystack = ks->keysock_keystack;

        ks3dbg(keystack, ("In keysock_wput\n"));

        switch (mp->b_datap->db_type) {
        case M_DATA:
                /*
                 * Silently discard.
                 */
                ks2dbg(keystack, ("raw M_DATA in keysock.\n"));
                freemsg(mp);
                return (0);
        case M_PROTO:
        case M_PCPROTO:
                if ((mp->b_wptr - rptr) >= sizeof (struct T_data_req)) {
                        if (((union T_primitives *)rptr)->type == T_DATA_REQ) {
                                if ((mp1 = mp->b_cont) == NULL) {
                                        /* No data after T_DATA_REQ. */
                                        ks2dbg(keystack,
                                            ("No data after DATA_REQ.\n"));
                                        freemsg(mp);
                                        return (0);
                                }
                                freeb(mp);
                                mp = mp1;
                                ks2dbg(keystack, ("T_DATA_REQ\n"));
                                break;  /* Out of switch. */
                        }
                }
                /* FALLTHRU */
        default:
                ks3dbg(keystack, ("In default wput case (%d %d).\n",
                    mp->b_datap->db_type, ((union T_primitives *)rptr)->type));
                keysock_wput_other(q, mp);
                return (0);
        }

        /* I now have a PF_KEY message in an M_DATA block, pointed to by mp. */
        keysock_parse(q, mp);
        return (0);
}

/* BELOW THIS LINE ARE ROUTINES INCLUDING AND RELATED TO keysock_rput(). */

/*
 * Called upon receipt of a KEYSOCK_HELLO_ACK to set up the appropriate
 * state vectors.
 */
static void
keysock_link_consumer(uint8_t satype, keysock_consumer_t *kc)
{
        keysock_t *ks;
        keysock_stack_t *keystack = kc->kc_keystack;

        mutex_enter(&keystack->keystack_consumers_lock);
        mutex_enter(&kc->kc_lock);
        if (keystack->keystack_consumers[satype] != NULL) {
                ks0dbg((
                    "Hmmmm, someone closed %d before the HELLO_ACK happened.\n",
                    satype));
                /*
                 * Perhaps updating the new below-me consumer with what I have
                 * so far would work too?
                 */
                mutex_exit(&kc->kc_lock);
                mutex_exit(&keystack->keystack_consumers_lock);
        } else {
                /* Add new below-me consumer. */
                keystack->keystack_consumers[satype] = kc;

                kc->kc_flags = 0;
                kc->kc_sa_type = satype;
                mutex_exit(&kc->kc_lock);
                mutex_exit(&keystack->keystack_consumers_lock);

                /* Scan the keysock list. */
                mutex_enter(&keystack->keystack_list_lock);
                for (ks = keystack->keystack_list; ks != NULL;
                    ks = ks->keysock_next) {
                        if (KEYSOCK_ISREG(ks, satype)) {
                                /*
                                 * XXX Perhaps send an SADB_REGISTER down on
                                 * the socket's behalf.
                                 */
                                ks1dbg(keystack,
                                    ("Socket %u registered already for "
                                    "new consumer.\n", ks->keysock_serial));
                        }
                }
                mutex_exit(&keystack->keystack_list_lock);
        }
}

/*
 * Generate a KEYSOCK_OUT_ERR message for my consumer.
 */
static void
keysock_out_err(keysock_consumer_t *kc, int ks_errno, mblk_t *mp)
{
        keysock_out_err_t *kse;
        mblk_t *imp;
        keysock_stack_t *keystack = kc->kc_keystack;

        imp = allocb(sizeof (ipsec_info_t), BPRI_HI);
        if (imp == NULL) {
                ks1dbg(keystack, ("keysock_out_err:  Can't alloc message.\n"));
                return;
        }

        imp->b_datap->db_type = M_CTL;
        imp->b_wptr += sizeof (ipsec_info_t);

        kse = (keysock_out_err_t *)imp->b_rptr;
        imp->b_cont = mp;
        kse->ks_err_type = KEYSOCK_OUT_ERR;
        kse->ks_err_len = sizeof (*kse);
        /* Is serial necessary? */
        kse->ks_err_serial = 0;
        kse->ks_err_errno = ks_errno;

        /*
         * XXX What else do I need to do here w.r.t. information
         * to tell the consumer what caused this error?
         *
         * I believe the answer is the PF_KEY ACQUIRE (or other) message
         * attached in mp, which is appended at the end.  I believe the
         * db_ref won't matter here, because the PF_KEY message is only read
         * for KEYSOCK_OUT_ERR.
         */

        putnext(kc->kc_wq, imp);
}

/* XXX this is a hack errno. */
#define EIPSECNOSA 255

/*
 * Route message (pointed by mp, header in samsg) toward appropriate
 * sockets.  Assume the message's creator did its job correctly.
 *
 * This should be a function that is followed by a return in its caller.
 * The compiler _should_ be able to use tail-call optimizations to make the
 * large ## of parameters not a huge deal.
 */
void
keysock_passup(mblk_t *mp, sadb_msg_t *samsg, minor_t serial,
    keysock_consumer_t *kc, boolean_t persistent, keysock_stack_t *keystack)
{
        keysock_t *ks;
        uint8_t satype = samsg->sadb_msg_satype;
        boolean_t toall = B_FALSE, allreg = B_FALSE, allereg = B_FALSE,
            setalg = B_FALSE;
        mblk_t *mp1;
        int err = EIPSECNOSA;

        /* Convert mp, which is M_DATA, into an M_PROTO of type T_DATA_IND */
        mp1 = allocb(sizeof (struct T_data_req), BPRI_HI);
        if (mp1 == NULL) {
                err = ENOMEM;
                goto error;
        }
        mp1->b_wptr += sizeof (struct T_data_req);
        ((struct T_data_ind *)mp1->b_rptr)->PRIM_type = T_DATA_IND;
        ((struct T_data_ind *)mp1->b_rptr)->MORE_flag = 0;
        mp1->b_datap->db_type = M_PROTO;
        mp1->b_cont = mp;
        mp = mp1;

        switch (samsg->sadb_msg_type) {
        case SADB_FLUSH:
        case SADB_GETSPI:
        case SADB_UPDATE:
        case SADB_X_UPDATEPAIR:
        case SADB_ADD:
        case SADB_DELETE:
        case SADB_X_DELPAIR:
        case SADB_EXPIRE:
                /*
                 * These are most likely replies.  Don't worry about
                 * KEYSOCK_OUT_ERR handling.  Deliver to all sockets.
                 */
                ks3dbg(keystack,
                    ("Delivering normal message (%d) to all sockets.\n",
                    samsg->sadb_msg_type));
                toall = B_TRUE;
                break;
        case SADB_REGISTER:
                /*
                 * REGISTERs come up for one of three reasons:
                 *
                 *      1.) In response to a normal SADB_REGISTER
                 *              (samsg->sadb_msg_satype != SADB_SATYPE_UNSPEC &&
                 *                  serial != 0)
                 *              Deliver to normal SADB_REGISTERed sockets.
                 *      2.) In response to an extended REGISTER
                 *              (samsg->sadb_msg_satype == SADB_SATYPE_UNSPEC)
                 *              Deliver to extended REGISTERed socket.
                 *      3.) Spontaneous algorithm changes
                 *              (samsg->sadb_msg_satype != SADB_SATYPE_UNSPEC &&
                 *                  serial == 0)
                 *              Deliver to REGISTERed sockets of all sorts.
                 */
                if (kc == NULL) {
                        /* Here because of keysock_error() call. */
                        ASSERT(samsg->sadb_msg_errno != 0);
                        break;  /* Out of switch. */
                }
                ks3dbg(keystack, ("Delivering REGISTER.\n"));
                if (satype == SADB_SATYPE_UNSPEC) {
                        /* REGISTER Reason #2 */
                        allereg = B_TRUE;
                        /*
                         * Rewhack SA type so PF_KEY socket holder knows what
                         * consumer generated this algorithm list.
                         */
                        satype = kc->kc_sa_type;
                        samsg->sadb_msg_satype = satype;
                        setalg = B_TRUE;
                } else if (serial == 0) {
                        /* REGISTER Reason #3 */
                        allreg = B_TRUE;
                        allereg = B_TRUE;
                } else {
                        /* REGISTER Reason #1 */
                        allreg = B_TRUE;
                        setalg = B_TRUE;
                }
                break;
        case SADB_ACQUIRE:
                /*
                 * ACQUIREs are either extended (sadb_msg_satype == 0) or
                 * regular (sadb_msg_satype != 0).  And we're guaranteed
                 * that serial == 0 for an ACQUIRE.
                 */
                ks3dbg(keystack, ("Delivering ACQUIRE.\n"));
                allereg = (satype == SADB_SATYPE_UNSPEC);
                allreg = !allereg;
                /*
                 * Corner case - if we send a regular ACQUIRE and there's
                 * extended ones registered, don't send an error down to
                 * consumers if nobody's listening and prematurely destroy
                 * their ACQUIRE record.  This might be too hackish of a
                 * solution.
                 */
                if (allreg && keystack->keystack_num_extended > 0)
                        err = 0;
                break;
        case SADB_X_PROMISC:
        case SADB_X_INVERSE_ACQUIRE:
        case SADB_DUMP:
        case SADB_GET:
        default:
                /*
                 * Deliver to the sender and promiscuous only.
                 */
                ks3dbg(keystack, ("Delivering sender/promisc only (%d).\n",
                    samsg->sadb_msg_type));
                break;
        }

        mutex_enter(&keystack->keystack_list_lock);
        for (ks = keystack->keystack_list; ks != NULL; ks = ks->keysock_next) {
                /* Delivery loop. */

                /*
                 * Check special keysock-setting cases (REGISTER replies)
                 * here.
                 */
                if (setalg && serial == ks->keysock_serial) {
                        ASSERT(kc != NULL);
                        ASSERT(kc->kc_sa_type == satype);
                        KEYSOCK_SETREG(ks, satype);
                }

                /*
                 * NOLOOP takes precedence over PROMISC.  So if you've set
                 * !SO_USELOOPBACK, don't expect to see any data...
                 */
                if (ks->keysock_flags & KEYSOCK_NOLOOP)
                        continue;

                /*
                 * Messages to all, or promiscuous sockets just GET the
                 * message.  Perform rules-type checking iff it's not for all
                 * listeners or the socket is in promiscuous mode.
                 *
                 * NOTE:Because of the (kc != NULL && ISREG()), make sure
                 *      extended ACQUIREs arrive off a consumer that is
                 *      part of the extended REGISTER set of consumers.
                 */
                if (serial != ks->keysock_serial &&
                    !toall &&
                    !(ks->keysock_flags & KEYSOCK_PROMISC) &&
                    !((ks->keysock_flags & KEYSOCK_EXTENDED) ?
                    allereg : allreg && kc != NULL &&
                    KEYSOCK_ISREG(ks, kc->kc_sa_type)))
                        continue;

                mp1 = dupmsg(mp);
                if (mp1 == NULL) {
                        ks2dbg(keystack, (
                            "keysock_passup():  dupmsg() failed.\n"));
                        mp1 = mp;
                        mp = NULL;
                        err = ENOMEM;
                }

                /*
                 * At this point, we can deliver or attempt to deliver
                 * this message.  We're free of obligation to report
                 * no listening PF_KEY sockets.  So set err to 0.
                 */
                err = 0;

                /*
                 * See if we canputnext(), as well as see if the message
                 * needs to be queued if we can't.
                 */
                if (!canputnext(ks->keysock_rq)) {
                        if (persistent) {
                                if (putq(ks->keysock_rq, mp1) == 0) {
                                        ks1dbg(keystack, (
                                            "keysock_passup: putq failed.\n"));
                                } else {
                                        continue;
                                }
                        }
                        freemsg(mp1);
                        continue;
                }

                ks3dbg(keystack,
                    ("Putting to serial %d.\n", ks->keysock_serial));
                /*
                 * Unlike the specific keysock instance case, this
                 * will only hit for listeners, so we will only
                 * putnext() if we can.
                 */
                putnext(ks->keysock_rq, mp1);
                if (mp == NULL)
                        break;  /* out of for loop. */
        }
        mutex_exit(&keystack->keystack_list_lock);

error:
        if ((err != 0) && (kc != NULL)) {
                /*
                 * Generate KEYSOCK_OUT_ERR for consumer.
                 * Basically, I send this back if I have not been able to
                 * transmit (for whatever reason)
                 */
                ks1dbg(keystack,
                    ("keysock_passup():  No registered of type %d.\n",
                    satype));
                if (mp != NULL) {
                        if (mp->b_datap->db_type == M_PROTO) {
                                mp1 = mp;
                                mp = mp->b_cont;
                                freeb(mp1);
                        }
                        /*
                         * Do a copymsg() because people who get
                         * KEYSOCK_OUT_ERR may alter the message contents.
                         */
                        mp1 = copymsg(mp);
                        if (mp1 == NULL) {
                                ks2dbg(keystack,
                                    ("keysock_passup: copymsg() failed.\n"));
                                mp1 = mp;
                                mp = NULL;
                        }
                        keysock_out_err(kc, err, mp1);
                }
        }

        /*
         * XXX Blank the message somehow.  This is difficult because we don't
         * know at this point if the message has db_ref > 1, etc.
         *
         * Optimally, keysock messages containing actual keying material would
         * be allocated with esballoc(), with a zeroing free function.
         */
        if (mp != NULL)
                freemsg(mp);
}

/*
 * Keysock's read service procedure is there only for PF_KEY reply
 * messages that really need to reach the top.
 */
static int
keysock_rsrv(queue_t *q)
{
        mblk_t *mp;

        while ((mp = getq(q)) != NULL) {
                if (canputnext(q)) {
                        putnext(q, mp);
                } else {
                        (void) putbq(q, mp);
                        return (0);
                }
        }
        return (0);
}

/*
 * The read procedure should only be invoked by a keysock consumer, like
 * ESP, AH, etc.  I should only see KEYSOCK_OUT and KEYSOCK_HELLO_ACK
 * messages on my read queues.
 */
static int
keysock_rput(queue_t *q, mblk_t *mp)
{
        keysock_consumer_t *kc = (keysock_consumer_t *)q->q_ptr;
        ipsec_info_t *ii;
        keysock_hello_ack_t *ksa;
        minor_t serial;
        mblk_t *mp1;
        sadb_msg_t *samsg;
        keysock_stack_t *keystack = kc->kc_keystack;

        /* Make sure I'm a consumer instance.  (i.e. something's below me) */
        ASSERT(WR(q)->q_next != NULL);

        if (mp->b_datap->db_type != M_CTL) {
                /*
                 * Keysock should only see keysock consumer interface
                 * messages (see ipsec_info.h) on its read procedure.
                 * To be robust, however, putnext() up so the STREAM head can
                 * deal with it appropriately.
                 */
                ks1dbg(keystack,
                    ("Hmmm, a non M_CTL (%d, 0x%x) on keysock_rput.\n",
                    mp->b_datap->db_type, mp->b_datap->db_type));
                putnext(q, mp);
                return (0);
        }

        ii = (ipsec_info_t *)mp->b_rptr;

        switch (ii->ipsec_info_type) {
        case KEYSOCK_OUT:
                /*
                 * A consumer needs to pass a response message or an ACQUIRE
                 * UP.  I assume that the consumer has done the right
                 * thing w.r.t. message creation, etc.
                 */
                serial = ((keysock_out_t *)mp->b_rptr)->ks_out_serial;
                mp1 = mp->b_cont;       /* Get M_DATA portion. */
                freeb(mp);
                samsg = (sadb_msg_t *)mp1->b_rptr;
                if (samsg->sadb_msg_type == SADB_FLUSH ||
                    (samsg->sadb_msg_type == SADB_DUMP &&
                    samsg->sadb_msg_len == SADB_8TO64(sizeof (*samsg)))) {
                        /*
                         * If I'm an end-of-FLUSH or an end-of-DUMP marker...
                         */
                        ASSERT(keystack->keystack_flushdump != 0);
                                                /* Am I flushing? */

                        mutex_enter(&kc->kc_lock);
                        kc->kc_flags &= ~KC_FLUSHING;
                        mutex_exit(&kc->kc_lock);

                        if (samsg->sadb_msg_errno != 0)
                                keystack->keystack_flushdump_errno =
                                    samsg->sadb_msg_errno;

                        /*
                         * Lower the atomic "flushing" count.  If it's
                         * the last one, send up the end-of-{FLUSH,DUMP} to
                         * the appropriate PF_KEY socket.
                         */
                        if (atomic_dec_32_nv(&keystack->keystack_flushdump) !=
                            0) {
                                ks1dbg(keystack,
                                    ("One flush/dump message back from %d,"
                                    " more to go.\n", samsg->sadb_msg_satype));
                                freemsg(mp1);
                                return (0);
                        }

                        samsg->sadb_msg_errno =
                            (uint8_t)keystack->keystack_flushdump_errno;
                        if (samsg->sadb_msg_type == SADB_DUMP) {
                                samsg->sadb_msg_seq = 0;
                        }
                }
                keysock_passup(mp1, samsg, serial, kc,
                    (samsg->sadb_msg_type == SADB_DUMP), keystack);
                return (0);
        case KEYSOCK_HELLO_ACK:
                /* Aha, now we can link in the consumer! */
                ksa = (keysock_hello_ack_t *)ii;
                keysock_link_consumer(ksa->ks_hello_satype, kc);
                freemsg(mp);
                return (0);
        default:
                ks1dbg(keystack, ("Hmmm, an IPsec info I'm not used to, 0x%x\n",
                    ii->ipsec_info_type));
                putnext(q, mp);
        }
        return (0);
}

/*
 * So we can avoid external linking problems....
 */
boolean_t
keysock_extended_reg(netstack_t *ns)
{
        keysock_stack_t *keystack = ns->netstack_keysock;

        return (keystack->keystack_num_extended != 0);
}

uint32_t
keysock_next_seq(netstack_t *ns)
{
        keysock_stack_t *keystack = ns->netstack_keysock;

        return (atomic_dec_32_nv(&keystack->keystack_acquire_seq));
}