/*
 * Copyright (c) 2011 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Coyote Point Systems, Inc.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

/*
 * Reduces the resources demanded by TCP sessions in TIME_WAIT-state using
 * methods called Vestigial Time-Wait (VTW) and Maximum Segment Lifetime
 * Truncation (MSLT).
 * 
 * MSLT and VTW were contributed by Coyote Point Systems, Inc.
 * 
 * Even after a TCP session enters the TIME_WAIT state, its corresponding
 * socket and protocol control blocks (PCBs) stick around until the TCP
 * Maximum Segment Lifetime (MSL) expires.  On a host whose workload
 * necessarily creates and closes down many TCP sockets, the sockets & PCBs
 * for TCP sessions in TIME_WAIT state amount to many megabytes of dead
 * weight in RAM.
 * 
 * Maximum Segment Lifetimes Truncation (MSLT) assigns each TCP session to
 * a class based on the nearness of the peer.  Corresponding to each class
 * is an MSL, and a session uses the MSL of its class.  The classes are
 * loopback (local host equals remote host), local (local host and remote
 * host are on the same link/subnet), and remote (local host and remote
 * host communicate via one or more gateways).  Classes corresponding to
 * nearer peers have lower MSLs by default: 2 seconds for loopback, 10
 * seconds for local, 60 seconds for remote.  Loopback and local sessions
 * expire more quickly when MSLT is used.
 * 
 * Vestigial Time-Wait (VTW) replaces a TIME_WAIT session's PCB/socket
 * dead weight with a compact representation of the session, called a
 * "vestigial PCB".  VTW data structures are designed to be very fast and
 * memory-efficient: for fast insertion and lookup of vestigial PCBs,
 * the PCBs are stored in a hash table that is designed to minimize the
 * number of cacheline visits per lookup/insertion.  The memory both
 * for vestigial PCBs and for elements of the PCB hashtable come from
 * fixed-size pools, and linked data structures exploit this to conserve
 * memory by representing references with a narrow index/offset from the
 * start of a pool instead of a pointer.  When space for new vestigial PCBs
 * runs out, VTW makes room by discarding old vestigial PCBs, oldest first.
 * VTW cooperates with MSLT.
 * 
 * It may help to think of VTW as a "FIN cache" by analogy to the SYN
 * cache.
 * 
 * A 2.8-GHz Pentium 4 running a test workload that creates TIME_WAIT
 * sessions as fast as it can is approximately 17% idle when VTW is active
 * versus 0% idle when VTW is inactive.  It has 103 megabytes more free RAM
 * when VTW is active (approximately 64k vestigial PCBs are created) than
 * when it is inactive.
 */

#include <sys/cdefs.h>

#ifdef _KERNEL_OPT
#include "opt_ddb.h"
#include "opt_inet.h"
#include "opt_inet_csum.h"
#include "opt_tcp_debug.h"
#endif

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kmem.h>
#include <sys/mbuf.h>
#include <sys/protosw.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
#include <sys/errno.h>
#include <sys/syslog.h>
#include <sys/pool.h>
#include <sys/domain.h>
#include <sys/kernel.h>
#include <net/if.h>
#include <net/if_types.h>

#include <netinet/in.h>
#include <netinet/in_systm.h>
#include <netinet/ip.h>
#include <netinet/in_pcb.h>
#include <netinet/in_var.h>
#include <netinet/ip_var.h>
#include <netinet/in_offload.h>
#include <netinet/ip6.h>
#include <netinet6/ip6_var.h>
#include <netinet6/in6_pcb.h>
#include <netinet6/ip6_var.h>
#include <netinet6/in6_var.h>
#include <netinet/icmp6.h>

#include <netinet/tcp.h>
#include <netinet/tcp_fsm.h>
#include <netinet/tcp_seq.h>
#include <netinet/tcp_timer.h>
#include <netinet/tcp_var.h>
#include <netinet/tcp_private.h>

#include <netinet/tcp_vtw.h>

__KERNEL_RCSID(0, "$NetBSD: tcp_vtw.c,v 1.25 2024/10/07 23:17:00 jakllsch Exp $");

#define db_trace(__a, __b)      do { } while (/*CONSTCOND*/0)

static void vtw_debug_init(void);

fatp_ctl_t fat_tcpv4;
fatp_ctl_t fat_tcpv6;
vtw_ctl_t  vtw_tcpv4[VTW_NCLASS];
vtw_ctl_t  vtw_tcpv6[VTW_NCLASS];
vtw_stats_t vtw_stats;

/* We provide state for the lookup_ports iterator.
 * As currently we are netlock-protected, there is one.
 * If we were finer-grain, we would have one per CPU.
 * I do not want to be in the business of alloc/free.
 * The best alternate would be allocate on the caller's
 * stack, but that would require them to know the struct,
 * or at least the size.
 * See how she goes.
 */
struct tcp_ports_iterator {
        union {
                struct in_addr  v4;
                struct in6_addr v6;
        }               addr;
        u_int           port;

        uint32_t        wild    : 1;

        vtw_ctl_t       *ctl;
        fatp_t          *fp;

        uint16_t        slot_idx;
        uint16_t        ctl_idx;
};

static struct tcp_ports_iterator tcp_ports_iterator_v4;
static struct tcp_ports_iterator tcp_ports_iterator_v6;

static int vtw_age(vtw_ctl_t *, struct timeval *);

/*!\brief allocate a fat pointer from a collection.
 */
static fatp_t *
fatp_alloc(fatp_ctl_t *fat)
{
        fatp_t  *fp     = 0;

        if (fat->nfree) {
                fp = fat->free;
                if (fp) {
                        fat->free = fatp_next(fat, fp);
                        --fat->nfree;
                        ++fat->nalloc;
                        fp->nxt = 0;

                        KASSERT(!fp->inuse);
                }
        }

        return fp;
}

/*!\brief free a fat pointer.
 */
static void
fatp_free(fatp_ctl_t *fat, fatp_t *fp)
{
        if (fp) {
                KASSERT(!fp->inuse);
                KASSERT(!fp->nxt);

                fp->nxt = fatp_index(fat, fat->free);
                fat->free = fp;

                ++fat->nfree;
                --fat->nalloc;
        }
}

/*!\brief initialise a collection of fat pointers.
 *
 *\param n      # hash buckets
 *\param m      total # fat pointers to allocate
 *
 * We allocate 2x as much, as we have two hashes: full and lport only.
 */
static void
fatp_init(fatp_ctl_t *fat, uint32_t n, uint32_t m,
    fatp_t *fat_base, fatp_t **fat_hash)
{
        fatp_t  *fp;

        KASSERT(n <= FATP_MAX / 2);

        fat->hash = fat_hash;
        fat->base = fat_base;

        fat->port = &fat->hash[m];

        fat->mask   = m - 1;    // ASSERT is power of 2 (m)
        fat->lim    = fat->base + 2*n - 1;
        fat->nfree  = 0;
        fat->nalloc = 2*n;

        /* Initialise the free list.
         */
        for (fp = fat->lim; fp >= fat->base; --fp) {
                fatp_free(fat, fp);
        }
}

/*
 * The `xtra' is XORed into the tag stored.
 */
static uint32_t fatp_xtra[] = {
        0x11111111,0x22222222,0x33333333,0x44444444,
        0x55555555,0x66666666,0x77777777,0x88888888,
        0x12121212,0x21212121,0x34343434,0x43434343,
        0x56565656,0x65656565,0x78787878,0x87878787,
        0x11221122,0x22112211,0x33443344,0x44334433,
        0x55665566,0x66556655,0x77887788,0x88778877,
        0x11112222,0x22221111,0x33334444,0x44443333,
        0x55556666,0x66665555,0x77778888,0x88887777,
};

/*!\brief turn a {fatp_t*,slot} into an integral key.
 *
 * The key can be used to obtain the fatp_t, and the slot,
 * as it directly encodes them.
 */
static inline uint32_t
fatp_key(fatp_ctl_t *fat, fatp_t *fp, uint32_t slot)
{
        CTASSERT(CACHE_LINE_SIZE == 32 ||
                 CACHE_LINE_SIZE == 64 ||
                 CACHE_LINE_SIZE == 128 ||
                 CACHE_LINE_SIZE == 256);

        switch (fatp_ntags()) {
        case 7:
                return (fatp_index(fat, fp) << 3) | slot;
        case 15:
                return (fatp_index(fat, fp) << 4) | slot;
        case 31:
                return (fatp_index(fat, fp) << 5) | slot;
        default:
                KASSERT(0 && "no support, for no good reason");
                return ~0;
        }
}

static inline uint32_t
fatp_slot_from_key(fatp_ctl_t *fat, uint32_t key)
{
        CTASSERT(CACHE_LINE_SIZE == 32 ||
                 CACHE_LINE_SIZE == 64 ||
                 CACHE_LINE_SIZE == 128 ||
                 CACHE_LINE_SIZE == 256);

        switch (fatp_ntags()) {
        case 7:
                return key & 7;
        case 15:
                return key & 15;
        case 31:
                return key & 31;
        default:
                KASSERT(0 && "no support, for no good reason");
                return ~0;
        }
}

static inline fatp_t *
fatp_from_key(fatp_ctl_t *fat, uint32_t key)
{
        CTASSERT(CACHE_LINE_SIZE == 32 ||
                 CACHE_LINE_SIZE == 64 ||
                 CACHE_LINE_SIZE == 128 ||
                 CACHE_LINE_SIZE == 256);

        switch (fatp_ntags()) {
        case 7:
                key >>= 3;
                break;
        case 15:
                key >>= 4;
                break;
        case 31:
                key >>= 5;
                break;
        default:
                KASSERT(0 && "no support, for no good reason");
                return 0;
        }

        return key ? fat->base + key - 1 : 0;
}

static inline uint32_t
idx_encode(vtw_ctl_t *ctl, uint32_t idx)
{
        return (idx << ctl->idx_bits) | idx;
}

static inline uint32_t
idx_decode(vtw_ctl_t *ctl, uint32_t bits)
{
        uint32_t        idx     = bits & ctl->idx_mask;

        if (idx_encode(ctl, idx) == bits)
                return idx;
        else
                return ~0;
}

/*!\brief       insert index into fatp hash
 *
 *\param        idx     -       index of element being placed in hash chain
 *\param        tag     -       32-bit tag identifier
 *
 *\returns
 *      value which can be used to locate entry.
 *
 *\note
 *      we rely on the fact that there are unused high bits in the index
 *      for verification purposes on lookup.
 */

static inline uint32_t
fatp_vtw_inshash(fatp_ctl_t *fat, uint32_t idx, uint32_t tag, int which,
    void *dbg)
{
        fatp_t  *fp;
        fatp_t  **hash = (which ? fat->port : fat->hash);
        int     i;

        fp = hash[tag & fat->mask];

        while (!fp || fatp_full(fp)) {
                fatp_t  *fq;

                /* All entries are inuse at the top level.
                 * We allocate a spare, and push the top level
                 * down one.  All entries in the fp we push down
                 * (think of a tape worm here) will be expelled sooner than
                 * any entries added subsequently to this hash bucket.
                 * This is a property of the time waits we are exploiting.
                 */

                fq = fatp_alloc(fat);
                if (!fq) {
                        vtw_age(fat->vtw, 0);
                        fp = hash[tag & fat->mask];
                        continue;
                }

                fq->inuse = 0;
                fq->nxt   = fatp_index(fat, fp);

                hash[tag & fat->mask] = fq;

                fp = fq;
        }

        KASSERT(!fatp_full(fp));

        /* Fill highest index first.  Lookup is lowest first.
         */
        for (i = fatp_ntags(); --i >= 0; ) {
                if (!((1 << i) & fp->inuse)) {
                        break;
                }
        }

        fp->inuse |= 1 << i;
        fp->tag[i] = tag ^ idx_encode(fat->vtw, idx) ^ fatp_xtra[i];

        db_trace(KTR_VTW
                 , (fp, "fat: inuse %5.5x tag[%x] %8.8x"
                    , fp->inuse
                    , i, fp->tag[i]));

        return fatp_key(fat, fp, i);
}

static inline int
vtw_alive(const vtw_t *vtw)
{
        return vtw->hashed && vtw->expire.tv_sec;
}

static inline uint32_t
vtw_index_v4(vtw_ctl_t *ctl, vtw_v4_t *v4)
{
        if (ctl->base.v4 <= v4 && v4 <= ctl->lim.v4)
                return v4 - ctl->base.v4;

        KASSERT(0 && "vtw out of bounds");

        return ~0;
}

static inline uint32_t
vtw_index_v6(vtw_ctl_t *ctl, vtw_v6_t *v6)
{
        if (ctl->base.v6 <= v6 && v6 <= ctl->lim.v6)
                return v6 - ctl->base.v6;

        KASSERT(0 && "vtw out of bounds");

        return ~0;
}

static inline uint32_t
vtw_index(vtw_ctl_t *ctl, vtw_t *vtw)
{
        if (ctl->clidx)
                ctl = ctl->ctl;

        if (ctl->is_v4)
                return vtw_index_v4(ctl, (vtw_v4_t *)vtw);

        if (ctl->is_v6)
                return vtw_index_v6(ctl, (vtw_v6_t *)vtw);

        KASSERT(0 && "neither 4 nor 6.  most curious.");

        return ~0;
}

static inline vtw_t *
vtw_from_index(vtw_ctl_t *ctl, uint32_t idx)
{
        if (ctl->clidx)
                ctl = ctl->ctl;

        /* See if the index looks like it might be an index.
         * Bits on outside of the valid index bits is a give away.
         */
        idx = idx_decode(ctl, idx);

        if (idx == ~0) {
                return 0;
        } else if (ctl->is_v4) {
                vtw_v4_t        *vtw = ctl->base.v4 + idx;

                return (ctl->base.v4 <= vtw && vtw <= ctl->lim.v4)
                        ? &vtw->common : 0;
        } else if (ctl->is_v6) {
                vtw_v6_t        *vtw = ctl->base.v6 + idx;

                return (ctl->base.v6 <= vtw && vtw <= ctl->lim.v6)
                        ? &vtw->common : 0;
        } else {
                KASSERT(0 && "badness");
                return 0;
        }
}

/*!\brief return the next vtw after this one.
 *
 * Due to the differing sizes of the entries in differing
 * arenas, we have to ensure we ++ the correct pointer type.
 *
 * Also handles wrap.
 */
static inline vtw_t *
vtw_next(vtw_ctl_t *ctl, vtw_t *vtw)
{
        if (ctl->is_v4) {
                vtw_v4_t        *v4 = (void*)vtw;

                vtw = &(++v4)->common;
        } else {
                vtw_v6_t        *v6 = (void*)vtw;

                vtw = &(++v6)->common;
        }

        if (vtw > ctl->lim.v)
                vtw = ctl->base.v;

        return vtw;
}

/*!\brief       remove entry from FATP hash chains
 */
static inline void
vtw_unhash(vtw_ctl_t *ctl, vtw_t *vtw)
{
        fatp_ctl_t      *fat    = ctl->fat;
        fatp_t          *fp;
        uint32_t        key = vtw->key;
        uint32_t        tag, slot, idx;
        vtw_v4_t        *v4 = (void*)vtw;
        vtw_v6_t        *v6 = (void*)vtw;

        if (!vtw->hashed) {
                KASSERT(0 && "unhashed");
                return;
        }

        if (fat->vtw->is_v4) {
                tag = v4_tag(v4->faddr, v4->fport, v4->laddr, v4->lport);
        } else if (fat->vtw->is_v6) {
                tag = v6_tag(&v6->faddr, v6->fport, &v6->laddr, v6->lport);
        } else {
                tag = 0;
                KASSERT(0 && "not reached");
        }

        /* Remove from fat->hash[]
         */
        slot = fatp_slot_from_key(fat, key);
        fp   = fatp_from_key(fat, key);
        idx  = vtw_index(ctl, vtw);

        db_trace(KTR_VTW
                 , (fp, "fat: del inuse %5.5x slot %x idx %x key %x tag %x"
                    , fp->inuse, slot, idx, key, tag));

        KASSERT(fp->inuse & (1 << slot));
        KASSERT(fp->tag[slot] == (tag ^ idx_encode(ctl, idx)
                                  ^ fatp_xtra[slot]));

        if ((fp->inuse & (1 << slot))
            && fp->tag[slot] == (tag ^ idx_encode(ctl, idx)
                                 ^ fatp_xtra[slot])) {
                fp->inuse ^= 1 << slot;
                fp->tag[slot] = 0;

                /* When we delete entries, we do not compact.  This is
                 * due to temporality.  We add entries, and they
                 * (eventually) expire. Older entries will be further
                 * down the chain.
                 */
                if (!fp->inuse) {
                        uint32_t hi = tag & fat->mask;
                        fatp_t  *fq = 0;
                        fatp_t  *fr = fat->hash[hi];

                        while (fr && fr != fp) {
                                fr = fatp_next(fat, fq = fr);
                        }

                        if (fr == fp) {
                                if (fq) {
                                        fq->nxt = fp->nxt;
                                        fp->nxt = 0;
                                        fatp_free(fat, fp);
                                } else {
                                        KASSERT(fat->hash[hi] == fp);

                                        if (fp->nxt) {
                                                fat->hash[hi]
                                                        = fatp_next(fat, fp);
                                                fp->nxt = 0;
                                                fatp_free(fat, fp);
                                        } else {
                                                /* retain for next use.
                                                 */
                                                ;
                                        }
                                }
                        } else {
                                fr = fat->hash[hi];

                                do {
                                        db_trace(KTR_VTW
                                                 , (fr
                                                    , "fat:*del inuse %5.5x"
                                                    " nxt %x"
                                                    , fr->inuse, fr->nxt));

                                        fr = fatp_next(fat, fq = fr);
                                } while (fr && fr != fp);

                                KASSERT(0 && "oops");
                        }
                }
                vtw->key ^= ~0;
        }
        
        if (fat->vtw->is_v4) {
                tag = v4_port_tag(v4->lport);
        } else if (fat->vtw->is_v6) {
                tag = v6_port_tag(v6->lport);
        }

        /* Remove from fat->port[]
         */
        key  = vtw->port_key;
        slot = fatp_slot_from_key(fat, key);
        fp   = fatp_from_key(fat, key);
        idx  = vtw_index(ctl, vtw);

        db_trace(KTR_VTW
                 , (fp, "fatport: del inuse %5.5x"
                    " slot %x idx %x key %x tag %x"
                    , fp->inuse, slot, idx, key, tag));

        KASSERT(fp->inuse & (1 << slot));
        KASSERT(fp->tag[slot] == (tag ^ idx_encode(ctl, idx)
                                  ^ fatp_xtra[slot]));

        if ((fp->inuse & (1 << slot))
            && fp->tag[slot] == (tag ^ idx_encode(ctl, idx)
                                 ^ fatp_xtra[slot])) {
                fp->inuse ^= 1 << slot;
                fp->tag[slot] = 0;

                if (!fp->inuse) {
                        uint32_t hi = tag & fat->mask;
                        fatp_t  *fq = 0;
                        fatp_t  *fr = fat->port[hi];

                        while (fr && fr != fp) {
                                fr = fatp_next(fat, fq = fr);
                        }

                        if (fr == fp) {
                                if (fq) {
                                        fq->nxt = fp->nxt;
                                        fp->nxt = 0;
                                        fatp_free(fat, fp);
                                } else {
                                        KASSERT(fat->port[hi] == fp);

                                        if (fp->nxt) {
                                                fat->port[hi]
                                                        = fatp_next(fat, fp);
                                                fp->nxt = 0;
                                                fatp_free(fat, fp);
                                        } else {
                                                /* retain for next use.
                                                 */
                                                ;
                                        }
                                }
                        }
                }
                vtw->port_key ^= ~0;
        }

        vtw->hashed = 0;
}

/*!\brief       remove entry from hash, possibly free.
 */
void
vtw_del(vtw_ctl_t *ctl, vtw_t *vtw)
{
        KASSERT(mutex_owned(softnet_lock));

        if (vtw->hashed) {
                ++vtw_stats.del;
                vtw_unhash(ctl, vtw);
        }

        /* We only delete the oldest entry.
         */
        if (vtw != ctl->oldest.v)
                return;

        --ctl->nalloc;
        ++ctl->nfree;

        vtw->expire.tv_sec  = 0;
        vtw->expire.tv_usec = ~0;

        if (!ctl->nalloc)
                ctl->oldest.v = 0;

        ctl->oldest.v = vtw_next(ctl, vtw);
}

/*!\brief       insert vestigial timewait in hash chain
 */
static void
vtw_inshash_v4(vtw_ctl_t *ctl, vtw_t *vtw)
{
        uint32_t        idx     = vtw_index(ctl, vtw);
        uint32_t        tag;
        vtw_v4_t        *v4 = (void*)vtw;

        KASSERT(mutex_owned(softnet_lock));
        KASSERT(!vtw->hashed);
        KASSERT(ctl->clidx == vtw->msl_class);

        ++vtw_stats.ins;

        tag = v4_tag(v4->faddr, v4->fport,
                     v4->laddr, v4->lport);

        vtw->key = fatp_vtw_inshash(ctl->fat, idx, tag, 0, vtw);

        db_trace(KTR_VTW, (ctl
                           , "vtw: ins %8.8x:%4.4x %8.8x:%4.4x"
                           " tag %8.8x key %8.8x"
                           , v4->faddr, v4->fport
                           , v4->laddr, v4->lport
                           , tag
                           , vtw->key));

        tag = v4_port_tag(v4->lport);
        vtw->port_key = fatp_vtw_inshash(ctl->fat, idx, tag, 1, vtw);

        db_trace(KTR_VTW, (ctl, "vtw: ins %P - %4.4x tag %8.8x key %8.8x"
                           , v4->lport, v4->lport
                           , tag
                           , vtw->key));

        vtw->hashed = 1;
}

/*!\brief       insert vestigial timewait in hash chain
 */
static void
vtw_inshash_v6(vtw_ctl_t *ctl, vtw_t *vtw)
{
        uint32_t        idx     = vtw_index(ctl, vtw);
        uint32_t        tag;
        vtw_v6_t        *v6     = (void*)vtw;

        KASSERT(mutex_owned(softnet_lock));
        KASSERT(!vtw->hashed);
        KASSERT(ctl->clidx == vtw->msl_class);

        ++vtw_stats.ins;

        tag = v6_tag(&v6->faddr, v6->fport,
                     &v6->laddr, v6->lport);

        vtw->key = fatp_vtw_inshash(ctl->fat, idx, tag, 0, vtw);

        tag = v6_port_tag(v6->lport);
        vtw->port_key = fatp_vtw_inshash(ctl->fat, idx, tag, 1, vtw);

        db_trace(KTR_VTW, (ctl, "vtw: ins %P - %4.4x tag %8.8x key %8.8x"
                           , v6->lport, v6->lport
                           , tag
                           , vtw->key));

        vtw->hashed = 1;
}

static vtw_t *
vtw_lookup_hash_v4(vtw_ctl_t *ctl, uint32_t faddr, uint16_t fport
                                 , uint32_t laddr, uint16_t lport
                                 , int which)
{
        vtw_v4_t        *v4;
        vtw_t           *vtw;
        uint32_t        tag;
        fatp_t          *fp;
        int             i;
        uint32_t        fatps = 0, probes = 0, losings = 0;

        if (!ctl || !ctl->fat)
                return 0;

        ++vtw_stats.look[which];

        if (which) {
                tag = v4_port_tag(lport);
                fp  = ctl->fat->port[tag & ctl->fat->mask];
        } else {
                tag = v4_tag(faddr, fport, laddr, lport);
                fp  = ctl->fat->hash[tag & ctl->fat->mask];
        }

        while (fp && fp->inuse) {
                uint32_t        inuse = fp->inuse;

                ++fatps;

                for (i = 0; inuse && i < fatp_ntags(); ++i) {
                        uint32_t        idx;

                        if (!(inuse & (1 << i)))
                                continue;

                        inuse ^= 1 << i;

                        ++probes;
                        ++vtw_stats.probe[which];

                        idx = fp->tag[i] ^ tag ^ fatp_xtra[i];
                        vtw = vtw_from_index(ctl, idx);

                        if (!vtw) {
                                /* Hopefully fast path.
                                 */
                                db_trace(KTR_VTW
                                         , (fp, "vtw: fast %A:%P %A:%P"
                                            " idx %x tag %x"
                                            , faddr, fport
                                            , laddr, lport
                                            , idx, tag));
                                continue;
                        }

                        v4 = (void*)vtw;

                        /* The de-referencing of vtw is what we want to avoid.
                         * Losing.
                         */
                        if (vtw_alive(vtw)
                            && ((which ? vtw->port_key : vtw->key)
                                == fatp_key(ctl->fat, fp, i))
                            && (which
                                || (v4->faddr == faddr && v4->laddr == laddr
                                    && v4->fport == fport))
                            && v4->lport == lport) {
                                ++vtw_stats.hit[which];

                                db_trace(KTR_VTW
                                         , (fp, "vtw: hit %8.8x:%4.4x"
                                            " %8.8x:%4.4x idx %x key %x"
                                            , faddr, fport
                                            , laddr, lport
                                            , idx_decode(ctl, idx), vtw->key));

                                KASSERT(vtw->hashed);

                                goto out;
                        }
                        ++vtw_stats.losing[which];
                        ++losings;
                        
                        if (vtw_alive(vtw)) {
                                db_trace(KTR_VTW
                                         , (fp, "vtw:!mis %8.8x:%4.4x"
                                            " %8.8x:%4.4x key %x tag %x"
                                            , faddr, fport
                                            , laddr, lport
                                            , fatp_key(ctl->fat, fp, i)
                                            , v4_tag(faddr, fport
                                                     , laddr, lport)));
                                db_trace(KTR_VTW
                                         , (vtw, "vtw:!mis %8.8x:%4.4x"
                                            " %8.8x:%4.4x key %x tag %x"
                                            , v4->faddr, v4->fport
                                            , v4->laddr, v4->lport
                                            , vtw->key
                                            , v4_tag(v4->faddr, v4->fport
                                                     , v4->laddr, v4->lport)));

                                if (vtw->key == fatp_key(ctl->fat, fp, i)) {
                                        db_trace(KTR_VTW
                                                 , (vtw, "vtw:!mis %8.8x:%4.4x"
                                                    " %8.8x:%4.4x key %x"
                                                    " which %x"
                                                    , v4->faddr, v4->fport
                                                    , v4->laddr, v4->lport
                                                    , vtw->key
                                                    , which));

                                } else {
                                        db_trace(KTR_VTW
                                                 , (vtw
                                                    , "vtw:!mis"
                                                    " key %8.8x != %8.8x"
                                                    " idx %x i %x which %x"
                                                    , vtw->key
                                                    , fatp_key(ctl->fat, fp, i)
                                                    , idx_decode(ctl, idx)
                                                    , i
                                                    , which));
                                }
                        } else {
                                db_trace(KTR_VTW
                                         , (fp
                                            , "vtw:!mis free entry"
                                            " idx %x vtw %p which %x"
                                            , idx_decode(ctl, idx)
                                            , vtw, which));
                        }
                }

                if (fp->nxt) {
                        fp = fatp_next(ctl->fat, fp);
                } else {
                        break;
                }
        }
        ++vtw_stats.miss[which];
        vtw = 0;
out:
        if (fatps > vtw_stats.max_chain[which])
                vtw_stats.max_chain[which] = fatps;
        if (probes > vtw_stats.max_probe[which])
                vtw_stats.max_probe[which] = probes;
        if (losings > vtw_stats.max_loss[which])
                vtw_stats.max_loss[which] = losings;

        return vtw;
}

static vtw_t *
vtw_lookup_hash_v6(vtw_ctl_t *ctl, const struct in6_addr *faddr, uint16_t fport
                                 , const struct in6_addr *laddr, uint16_t lport
                                 , int which)
{
        vtw_v6_t        *v6;
        vtw_t           *vtw;
        uint32_t        tag;
        fatp_t          *fp;
        int             i;
        uint32_t        fatps = 0, probes = 0, losings = 0;

        ++vtw_stats.look[which];

        if (!ctl || !ctl->fat)
                return 0;

        if (which) {
                tag = v6_port_tag(lport);
                fp  = ctl->fat->port[tag & ctl->fat->mask];
        } else {
                tag = v6_tag(faddr, fport, laddr, lport);
                fp  = ctl->fat->hash[tag & ctl->fat->mask];
        }

        while (fp && fp->inuse) {
                uint32_t        inuse = fp->inuse;

                ++fatps;

                for (i = 0; inuse && i < fatp_ntags(); ++i) {
                        uint32_t        idx;

                        if (!(inuse & (1 << i)))
                                continue;

                        inuse ^= 1 << i;

                        ++probes;
                        ++vtw_stats.probe[which];

                        idx = fp->tag[i] ^ tag ^ fatp_xtra[i];
                        vtw = vtw_from_index(ctl, idx);

                        db_trace(KTR_VTW
                                 , (fp, "probe: %2d %6A:%4.4x %6A:%4.4x idx %x"
                                    , i
                                    , db_store(faddr, sizeof (*faddr)), fport
                                    , db_store(laddr, sizeof (*laddr)), lport
                                    , idx_decode(ctl, idx)));

                        if (!vtw) {
                                /* Hopefully fast path.
                                 */
                                continue;
                        }

                        v6 = (void*)vtw;

                        if (vtw_alive(vtw)
                            && ((which ? vtw->port_key : vtw->key)
                                == fatp_key(ctl->fat, fp, i))
                            && v6->lport == lport
                            && (which
                                || (v6->fport == fport
                                    && !bcmp(&v6->faddr, faddr, sizeof (*faddr))
                                    && !bcmp(&v6->laddr, laddr
                                             , sizeof (*laddr))))) {
                                ++vtw_stats.hit[which];

                                KASSERT(vtw->hashed);
                                goto out;
                        } else {
                                ++vtw_stats.losing[which];
                                ++losings;
                        }
                }

                if (fp->nxt) {
                        fp = fatp_next(ctl->fat, fp);
                } else {
                        break;
                }
        }
        ++vtw_stats.miss[which];
        vtw = 0;
out:
        if (fatps > vtw_stats.max_chain[which])
                vtw_stats.max_chain[which] = fatps;
        if (probes > vtw_stats.max_probe[which])
                vtw_stats.max_probe[which] = probes;
        if (losings > vtw_stats.max_loss[which])
                vtw_stats.max_loss[which] = losings;

        return vtw;
}

/*!\brief port iterator
 */
static vtw_t *
vtw_next_port_v4(struct tcp_ports_iterator *it)
{
        vtw_ctl_t       *ctl = it->ctl;
        vtw_v4_t        *v4;
        vtw_t           *vtw;
        uint32_t        tag;
        uint16_t        lport = it->port;
        fatp_t          *fp;
        int             i;
        uint32_t        fatps = 0, probes = 0, losings = 0;

        tag = v4_port_tag(lport);
        if (!it->fp) {
                it->fp = ctl->fat->port[tag & ctl->fat->mask];
                it->slot_idx = 0;
        }
        fp  = it->fp;

        while (fp) {
                uint32_t        inuse = fp->inuse;

                ++fatps;

                for (i = it->slot_idx; inuse && i < fatp_ntags(); ++i) {
                        uint32_t        idx;

                        if (!(inuse & (1 << i)))
                                continue;

                        inuse &= ~0U << i;

                        if (i < it->slot_idx)
                                continue;

                        ++vtw_stats.probe[1];
                        ++probes;

                        idx = fp->tag[i] ^ tag ^ fatp_xtra[i];
                        vtw = vtw_from_index(ctl, idx);

                        if (!vtw) {
                                /* Hopefully fast path.
                                 */
                                continue;
                        }

                        v4 = (void*)vtw;

                        if (vtw_alive(vtw)
                            && vtw->port_key == fatp_key(ctl->fat, fp, i)
                            && v4->lport == lport) {
                                ++vtw_stats.hit[1];

                                it->slot_idx = i + 1;

                                goto out;
                        } else if (vtw_alive(vtw)) {
                                ++vtw_stats.losing[1];
                                ++losings;

                                db_trace(KTR_VTW
                                         , (vtw, "vtw:!mis"
                                            " port %8.8x:%4.4x %8.8x:%4.4x"
                                            " key %x port %x"
                                            , v4->faddr, v4->fport
                                            , v4->laddr, v4->lport
                                            , vtw->key
                                            , lport));
                        } else {
                                /* Really losing here.  We are coming
                                 * up with references to free entries.
                                 * Might find it better to use
                                 * traditional, or need another
                                 * add-hockery.  The other add-hockery
                                 * would be to pul more into into the
                                 * cache line to reject the false
                                 * hits.
                                 */
                                ++vtw_stats.losing[1];
                                ++losings;
                                db_trace(KTR_VTW
                                         , (fp, "vtw:!mis port %x"
                                            " - free entry idx %x vtw %p"
                                            , lport
                                            , idx_decode(ctl, idx)
                                            , vtw));
                        }
                }

                if (fp->nxt) {
                        it->fp = fp = fatp_next(ctl->fat, fp);
                        it->slot_idx = 0;
                } else {
                        it->fp = 0;
                        break;
                }
        }
        ++vtw_stats.miss[1];

        vtw = 0;
out:
        if (fatps > vtw_stats.max_chain[1])
                vtw_stats.max_chain[1] = fatps;
        if (probes > vtw_stats.max_probe[1])
                vtw_stats.max_probe[1] = probes;
        if (losings > vtw_stats.max_loss[1])
                vtw_stats.max_loss[1] = losings;

        return vtw;
}

/*!\brief port iterator
 */
static vtw_t *
vtw_next_port_v6(struct tcp_ports_iterator *it)
{
        vtw_ctl_t       *ctl = it->ctl;
        vtw_v6_t        *v6;
        vtw_t           *vtw;
        uint32_t        tag;
        uint16_t        lport = it->port;
        fatp_t          *fp;
        int             i;
        uint32_t        fatps = 0, probes = 0, losings = 0;

        tag = v6_port_tag(lport);
        if (!it->fp) {
                it->fp = ctl->fat->port[tag & ctl->fat->mask];
                it->slot_idx = 0;
        }
        fp  = it->fp;

        while (fp) {
                uint32_t        inuse = fp->inuse;

                ++fatps;

                for (i = it->slot_idx; inuse && i < fatp_ntags(); ++i) {
                        uint32_t        idx;

                        if (!(inuse & (1 << i)))
                                continue;

                        inuse &= ~0U << i;

                        if (i < it->slot_idx)
                                continue;

                        ++vtw_stats.probe[1];
                        ++probes;

                        idx = fp->tag[i] ^ tag ^ fatp_xtra[i];
                        vtw = vtw_from_index(ctl, idx);

                        if (!vtw) {
                                /* Hopefully fast path.
                                 */
                                continue;
                        }

                        v6 = (void*)vtw;

                        db_trace(KTR_VTW
                                 , (vtw, "vtw: i %x idx %x fp->tag %x"
                                    " tag %x xtra %x"
                                    , i, idx_decode(ctl, idx)
                                    , fp->tag[i], tag, fatp_xtra[i]));

                        if (vtw_alive(vtw)
                            && vtw->port_key == fatp_key(ctl->fat, fp, i)
                            && v6->lport == lport) {
                                ++vtw_stats.hit[1];

                                db_trace(KTR_VTW
                                         , (fp, "vtw: nxt port %P - %4.4x"
                                            " idx %x key %x"
                                            , lport, lport
                                            , idx_decode(ctl, idx), vtw->key));

                                it->slot_idx = i + 1;
                                goto out;
                        } else if (vtw_alive(vtw)) {
                                ++vtw_stats.losing[1];

                                db_trace(KTR_VTW
                                         , (vtw, "vtw:!mis port %6A:%4.4x"
                                            " %6A:%4.4x key %x port %x"
                                            , db_store(&v6->faddr
                                                       , sizeof (v6->faddr))
                                            , v6->fport
                                            , db_store(&v6->laddr
                                                       , sizeof (v6->faddr))
                                            , v6->lport
                                            , vtw->key
                                            , lport));
                        } else {
                                /* Really losing here.  We are coming
                                 * up with references to free entries.
                                 * Might find it better to use
                                 * traditional, or need another
                                 * add-hockery.  The other add-hockery
                                 * would be to pul more into into the
                                 * cache line to reject the false
                                 * hits.
                                 */
                                ++vtw_stats.losing[1];
                                ++losings;

                                db_trace(KTR_VTW
                                         , (fp
                                            , "vtw:!mis port %x"
                                            " - free entry idx %x vtw %p"
                                            , lport, idx_decode(ctl, idx)
                                            , vtw));
                        }
                }

                if (fp->nxt) {
                        it->fp = fp = fatp_next(ctl->fat, fp);
                        it->slot_idx = 0;
                } else {
                        it->fp = 0;
                        break;
                }
        }
        ++vtw_stats.miss[1];

        vtw = 0;
out:
        if (fatps > vtw_stats.max_chain[1])
                vtw_stats.max_chain[1] = fatps;
        if (probes > vtw_stats.max_probe[1])
                vtw_stats.max_probe[1] = probes;
        if (losings > vtw_stats.max_loss[1])
                vtw_stats.max_loss[1] = losings;

        return vtw;
}

/*!\brief initialise the VTW allocation arena
 *
 * There are 1+3 allocation classes:
 *      0       classless
 *      {1,2,3} MSL-class based allocation
 *
 * The allocation arenas are all initialised.  Classless gets all the
 * space.  MSL-class based divides the arena, so that allocation
 * within a class can proceed without having to consider entries
 * (aka: cache lines) from different classes.
 *
 * Usually, we are completely classless or class-based, but there can be
 * transition periods, corresponding to dynamic adjustments in the config
 * by the operator.
 */
static void
vtw_init(fatp_ctl_t *fat, vtw_ctl_t *ctl, const uint32_t n, vtw_t *ctl_base_v)
{
        int class_n, i;
        vtw_t   *base;

        ctl->base.v = ctl_base_v;

        if (ctl->is_v4) {
                ctl->lim.v4    = ctl->base.v4 + n - 1;
                ctl->alloc.v4  = ctl->base.v4;
        } else {
                ctl->lim.v6    = ctl->base.v6 + n - 1;
                ctl->alloc.v6  = ctl->base.v6;
        }

        ctl->nfree  = n;
        ctl->ctl    = ctl;

        ctl->idx_bits = 32;
        for (ctl->idx_mask = ~0; (ctl->idx_mask & (n-1)) == n-1; ) {
                ctl->idx_mask >>= 1;
                ctl->idx_bits  -= 1;
        }

        ctl->idx_mask <<= 1;
        ctl->idx_mask  |= 1;
        ctl->idx_bits  += 1;

        ctl->fat = fat;
        fat->vtw = ctl;

        /* Divide the resources equally amongst the classes.
         * This is not optimal, as the different classes
         * arrive and leave at different rates, but it is
         * the best I can do for now.
         */
        class_n = n / (VTW_NCLASS-1);
        base    = ctl->base.v;

        for (i = 1; i < VTW_NCLASS; ++i) {
                int j;

                ctl[i] = ctl[0];
                ctl[i].clidx = i;

                ctl[i].base.v = base;
                ctl[i].alloc  = ctl[i].base;

                for (j = 0; j < class_n - 1; ++j) {
                        if (tcp_msl_enable)
                                base->msl_class = i;
                        base = vtw_next(ctl, base);
                }

                ctl[i].lim.v = base;
                base = vtw_next(ctl, base);
                ctl[i].nfree = class_n;
        }

        vtw_debug_init();
}

/*!\brief       map class to TCP MSL
 */
static inline uint32_t
class_to_msl(int msl_class)
{
        switch (msl_class) {
        case 0:
        case 1:
                return tcp_msl_remote ? tcp_msl_remote : (TCPTV_MSL >> 0);
        case 2:
                return tcp_msl_local ? tcp_msl_local : (TCPTV_MSL >> 1);
        default:
                return tcp_msl_loop ? tcp_msl_loop : (TCPTV_MSL >> 2);
        }
}

/*!\brief       map TCP MSL to class
 */
static inline uint32_t
msl_to_class(int msl)
{
        if (tcp_msl_enable) {
                if (msl <= (tcp_msl_loop ? tcp_msl_loop : (TCPTV_MSL >> 2)))
                        return 1+2;
                if (msl <= (tcp_msl_local ? tcp_msl_local : (TCPTV_MSL >> 1)))
                        return 1+1;
                return 1;
        }
        return 0;
}

/*!\brief allocate a vtw entry
 */
static inline vtw_t *
vtw_alloc(vtw_ctl_t *ctl)
{
        vtw_t   *vtw    = 0;
        int     stuck   = 0;
        int     avail   = ctl ? (ctl->nalloc + ctl->nfree) : 0;
        int     msl;

        KASSERT(mutex_owned(softnet_lock));

        /* If no resources, we will not get far.
         */
        if (!ctl || !ctl->base.v4 || avail <= 0)
                return 0;

        /* Obtain a free one.
         */
        while (!ctl->nfree) {
                vtw_age(ctl, 0);

                if (++stuck > avail) {
                        /* When in transition between
                         * schemes (classless, classed) we
                         * can be stuck having to await the
                         * expiration of cross-allocated entries.
                         *
                         * Returning zero means we will fall back to the
                         * traditional TIME_WAIT handling, except in the
                         * case of a re-shed, in which case we cannot
                         * perform the reshecd, but will retain the extant
                         * entry.
                         */
                        db_trace(KTR_VTW
                                 , (ctl, "vtw:!none free in class %x %x/%x"
                                    , ctl->clidx
                                    , ctl->nalloc, ctl->nfree));

                        return 0;
                }
        }

        vtw = ctl->alloc.v;

        if (vtw->msl_class != ctl->clidx) {
                /* Usurping rules:
                 *      0 -> {1,2,3} or {1,2,3} -> 0
                 */
                KASSERT(!vtw->msl_class || !ctl->clidx);

                if (vtw->hashed || vtw->expire.tv_sec) {
                    /* As this is owned by some other class,
                     * we must wait for it to expire it.
                     * This will only happen on class/classless
                     * transitions, which are guaranteed to progress
                     * to completion in small finite time, barring bugs.
                     */
                    db_trace(KTR_VTW
                             , (ctl, "vtw:!%p class %x!=%x %x:%x%s"
                                , vtw, vtw->msl_class, ctl->clidx
                                , vtw->expire.tv_sec
                                , vtw->expire.tv_usec
                                , vtw->hashed ? " hashed" : ""));

                    return 0;
                }

                db_trace(KTR_VTW
                         , (ctl, "vtw:!%p usurped from %x to %x"
                            , vtw, vtw->msl_class, ctl->clidx));

                vtw->msl_class = ctl->clidx;
        }

        if (vtw_alive(vtw)) {
                KASSERT(0 && "next free not free");
                return 0;
        }

        /* Advance allocation pointer.
         */
        ctl->alloc.v = vtw_next(ctl, vtw);

        --ctl->nfree;
        ++ctl->nalloc;

        msl = (2 * class_to_msl(ctl->clidx) * 1000) / PR_SLOWHZ;        // msec

        /* mark expiration
         */
        getmicrouptime(&vtw->expire);

        /* Move expiration into the future.
         */
        vtw->expire.tv_sec  += msl / 1000;
        vtw->expire.tv_usec += 1000 * (msl % 1000);

        while (vtw->expire.tv_usec >= 1000*1000) {
                vtw->expire.tv_usec -= 1000*1000;
                vtw->expire.tv_sec  += 1;
        }

        if (!ctl->oldest.v)
                ctl->oldest.v = vtw;

        return vtw;
}

/*!\brief expiration
 */
static int
vtw_age(vtw_ctl_t *ctl, struct timeval *_when)
{
        vtw_t   *vtw;
        struct timeval then, *when = _when;
        int     maxtries = 0;

        if (!ctl->oldest.v) {
                KASSERT(!ctl->nalloc);
                return 0;
        }

        for (vtw = ctl->oldest.v; vtw && ctl->nalloc; ) {
                if (++maxtries > ctl->nalloc)
                        break;

                if (vtw->msl_class != ctl->clidx) {
                        db_trace(KTR_VTW
                                 , (vtw, "vtw:!age class mismatch %x != %x"
                                    , vtw->msl_class, ctl->clidx));
                        /* XXXX
                         * See if the appropriate action is to skip to the next.
                         * XXXX
                         */
                        ctl->oldest.v = vtw = vtw_next(ctl, vtw);
                        continue;
                }
                if (!when) {
                        /* Latch oldest timeval if none specified.
                         */
                        then = vtw->expire;
                        when = &then;
                }

                if (!timercmp(&vtw->expire, when, <=))
                        break;

                db_trace(KTR_VTW
                         , (vtw, "vtw: expire %x %8.8x:%8.8x %x/%x"
                            , ctl->clidx
                            , vtw->expire.tv_sec
                            , vtw->expire.tv_usec
                            , ctl->nalloc
                            , ctl->nfree));

                if (!_when)
                        ++vtw_stats.kill;

                vtw_del(ctl, vtw);
                vtw = ctl->oldest.v;
        }

        return ctl->nalloc;     // # remaining allocated
}

static callout_t vtw_cs;

/*!\brief notice the passage of time.
 * It seems to be getting faster.  What happened to the year?
 */
static void
vtw_tick(void *arg)
{
        struct timeval now;
        int i, cnt = 0;

        getmicrouptime(&now);

        db_trace(KTR_VTW, (arg, "vtk: tick - now %8.8x:%8.8x"
                           , now.tv_sec, now.tv_usec));

        mutex_enter(softnet_lock);

        for (i = 0; i < VTW_NCLASS; ++i) {
                cnt += vtw_age(&vtw_tcpv4[i], &now);
                cnt += vtw_age(&vtw_tcpv6[i], &now);
        }

        /* Keep ticks coming while we need them.
         */
        if (cnt)
                callout_schedule(&vtw_cs, hz / 5);
        else {
                tcp_vtw_was_enabled = 0;
                tcbtable.vestige    = 0;
        }
        mutex_exit(softnet_lock);
}

/* inpcb_lookup_locals assist for handling vestigial entries.
 */
static void *
tcp_init_ports_v4(struct in_addr addr, u_int port, int wild)
{
        struct tcp_ports_iterator *it = &tcp_ports_iterator_v4;

        bzero(it, sizeof (*it));

        /* Note: the reference to vtw_tcpv4[0] is fine.
         * We do not need per-class iteration.  We just
         * need to get to the fat, and there is one
         * shared fat.
         */
        if (vtw_tcpv4[0].fat) {
                it->addr.v4 = addr;
                it->port = port;
                it->wild = !!wild;
                it->ctl  = &vtw_tcpv4[0];

                ++vtw_stats.look[1];
        }

        return it;
}

/*!\brief export an IPv4 vtw.
 */
static int
vtw_export_v4(vtw_ctl_t *ctl, vtw_t *vtw, vestigial_inpcb_t *res)
{
        vtw_v4_t        *v4 = (void*)vtw;

        bzero(res, sizeof (*res));

        if (ctl && vtw) {
                if (!ctl->clidx && vtw->msl_class)
                        ctl += vtw->msl_class;
                else
                        KASSERT(ctl->clidx == vtw->msl_class);

                res->valid = 1;
                res->v4    = 1;

                res->faddr.v4.s_addr = v4->faddr;
                res->laddr.v4.s_addr = v4->laddr;
                res->fport      = v4->fport;
                res->lport      = v4->lport;
                res->vtw        = vtw;          // netlock held over call(s)
                res->ctl        = ctl;
                res->reuse_addr = vtw->reuse_addr;
                res->reuse_port = vtw->reuse_port;
                res->snd_nxt    = vtw->snd_nxt;
                res->rcv_nxt    = vtw->rcv_nxt;
                res->rcv_wnd    = vtw->rcv_wnd;
                res->uid        = vtw->uid;
        }

        return res->valid;
}

/*!\brief return next port in the port iterator.  yowza.
 */
static int
tcp_next_port_v4(void *arg, struct vestigial_inpcb *res)
{
        struct tcp_ports_iterator *it = arg;
        vtw_t           *vtw = 0;

        if (it->ctl)
                vtw = vtw_next_port_v4(it);

        if (!vtw)
                it->ctl = 0;

        return vtw_export_v4(it->ctl, vtw, res);
}

static int
tcp_lookup_v4(struct in_addr faddr, uint16_t fport,
              struct in_addr laddr, uint16_t lport,
              struct vestigial_inpcb *res)
{
        vtw_t           *vtw;
        vtw_ctl_t       *ctl;


        db_trace(KTR_VTW
                 , (res, "vtw: lookup %A:%P %A:%P"
                    , faddr, fport
                    , laddr, lport));

        vtw = vtw_lookup_hash_v4((ctl = &vtw_tcpv4[0])
                                 , faddr.s_addr, fport
                                 , laddr.s_addr, lport, 0);

        return vtw_export_v4(ctl, vtw, res);
}

/* inpcb_lookup_locals assist for handling vestigial entries.
 */
static void *
tcp_init_ports_v6(const struct in6_addr *addr, u_int port, int wild)
{
        struct tcp_ports_iterator *it = &tcp_ports_iterator_v6;

        bzero(it, sizeof (*it));

        /* Note: the reference to vtw_tcpv6[0] is fine.
         * We do not need per-class iteration.  We just
         * need to get to the fat, and there is one
         * shared fat.
         */
        if (vtw_tcpv6[0].fat) {
                it->addr.v6 = *addr;
                it->port = port;
                it->wild = !!wild;
                it->ctl  = &vtw_tcpv6[0];

                ++vtw_stats.look[1];
        }

        return it;
}

/*!\brief export an IPv6 vtw.
 */
static int
vtw_export_v6(vtw_ctl_t *ctl, vtw_t *vtw, vestigial_inpcb_t *res)
{
        vtw_v6_t        *v6 = (void*)vtw;

        bzero(res, sizeof (*res));

        if (ctl && vtw) {
                if (!ctl->clidx && vtw->msl_class)
                        ctl += vtw->msl_class;
                else
                        KASSERT(ctl->clidx == vtw->msl_class);

                res->valid = 1;
                res->v4    = 0;

                res->faddr.v6   = v6->faddr;
                res->laddr.v6   = v6->laddr;
                res->fport      = v6->fport;
                res->lport      = v6->lport;
                res->vtw        = vtw;          // netlock held over call(s)
                res->ctl        = ctl;

                res->v6only     = vtw->v6only;
                res->reuse_addr = vtw->reuse_addr;
                res->reuse_port = vtw->reuse_port;

                res->snd_nxt    = vtw->snd_nxt;
                res->rcv_nxt    = vtw->rcv_nxt;
                res->rcv_wnd    = vtw->rcv_wnd;
                res->uid        = vtw->uid;
        }

        return res->valid;
}

static int
tcp_next_port_v6(void *arg, struct vestigial_inpcb *res)
{
        struct tcp_ports_iterator *it = arg;
        vtw_t           *vtw = 0;

        if (it->ctl)
                vtw = vtw_next_port_v6(it);

        if (!vtw)
                it->ctl = 0;

        return vtw_export_v6(it->ctl, vtw, res);
}

static int
tcp_lookup_v6(const struct in6_addr *faddr, uint16_t fport,
              const struct in6_addr *laddr, uint16_t lport,
              struct vestigial_inpcb *res)
{
        vtw_ctl_t       *ctl;
        vtw_t           *vtw;

        db_trace(KTR_VTW
                 , (res, "vtw: lookup %6A:%P %6A:%P"
                    , db_store(faddr, sizeof (*faddr)), fport
                    , db_store(laddr, sizeof (*laddr)), lport));

        vtw = vtw_lookup_hash_v6((ctl = &vtw_tcpv6[0])
                                 , faddr, fport
                                 , laddr, lport, 0);

        return vtw_export_v6(ctl, vtw, res);
}

static vestigial_hooks_t tcp_hooks = {
        .init_ports4    = tcp_init_ports_v4,
        .next_port4     = tcp_next_port_v4,
        .lookup4        = tcp_lookup_v4,
        .init_ports6    = tcp_init_ports_v6,
        .next_port6     = tcp_next_port_v6,
        .lookup6        = tcp_lookup_v6,
};

static bool
vtw_select(int af, fatp_ctl_t **fatp, vtw_ctl_t **ctlp)
{
        fatp_ctl_t      *fat;
        vtw_ctl_t       *ctl;

        switch (af) {
        case AF_INET:
                fat = &fat_tcpv4;
                ctl = &vtw_tcpv4[0];
                break;
        case AF_INET6:
                fat = &fat_tcpv6;
                ctl = &vtw_tcpv6[0];
                break;
        default:
                return false;
        }
        if (fatp != NULL)
                *fatp = fat;
        if (ctlp != NULL)
                *ctlp = ctl;
        return true;
}

/*!\brief       initialize controlling instance
 */
static int
vtw_control_init(int af)
{
        fatp_ctl_t      *fat;
        vtw_ctl_t       *ctl;
        fatp_t          *fat_base;
        fatp_t          **fat_hash;
        vtw_t           *ctl_base_v;
        uint32_t        n, m;
        size_t sz;

        KASSERT(powerof2(tcp_vtw_entries));

        if (!vtw_select(af, &fat, &ctl))
                return EAFNOSUPPORT;

        if (fat->hash != NULL) {
                KASSERT(fat->base != NULL && ctl->base.v != NULL);
                return 0;
        }

        /* Allocate 10% more capacity in the fat pointers.
         * We should only need ~#hash additional based on
         * how they age, but TIME_WAIT assassination could cause
         * sparse fat pointer utilisation.
         */
        m = 512;
        n = 2*m + (11 * (tcp_vtw_entries / fatp_ntags())) / 10;
        sz = (ctl->is_v4 ? sizeof(vtw_v4_t) : sizeof(vtw_v6_t));

        fat_hash = kmem_zalloc(2*m * sizeof(fatp_t *), KM_SLEEP);
        fat_base = kmem_zalloc(2*n * sizeof(fatp_t), KM_SLEEP);
        ctl_base_v = kmem_zalloc(tcp_vtw_entries * sz, KM_SLEEP);
        fatp_init(fat, n, m, fat_base, fat_hash);
        vtw_init(fat, ctl, tcp_vtw_entries, ctl_base_v);

        return 0;
}

/*!\brief       select controlling instance
 */
static vtw_ctl_t *
vtw_control(int af, uint32_t msl)
{
        fatp_ctl_t      *fat;
        vtw_ctl_t       *ctl;
        int             msl_class = msl_to_class(msl);

        if (!vtw_select(af, &fat, &ctl))
                return NULL;

        if (!fat->base || !ctl->base.v)
                return NULL;

        if (!tcp_vtw_was_enabled) {
                /* This guarantees is timer ticks until we no longer need them.
                 */
                tcp_vtw_was_enabled = 1;

                callout_schedule(&vtw_cs, hz / 5);

                tcbtable.vestige = &tcp_hooks;
        }

        return ctl + msl_class;
}

/*!\brief       add TCP pcb to vestigial timewait
 */
int
vtw_add(int af, struct tcpcb *tp)
{
#ifdef VTW_DEBUG
        int             enable;
#endif
        vtw_ctl_t       *ctl;
        vtw_t           *vtw;

        KASSERT(mutex_owned(softnet_lock));

        ctl = vtw_control(af, tp->t_msl);
        if (!ctl)
                return 0;

#ifdef VTW_DEBUG
        enable = (af == AF_INET) ? tcp4_vtw_enable : tcp6_vtw_enable;
#endif

        vtw = vtw_alloc(ctl);

        if (vtw) {
                vtw->snd_nxt = tp->snd_nxt;
                vtw->rcv_nxt = tp->rcv_nxt;

                switch (af) {
                case AF_INET: {
                        struct inpcb    *inp = tp->t_inpcb;
                        vtw_v4_t        *v4  = (void*)vtw;

                        v4->faddr = in4p_faddr(inp).s_addr;
                        v4->laddr = in4p_laddr(inp).s_addr;
                        v4->fport = inp->inp_fport;
                        v4->lport = inp->inp_lport;

                        vtw->reuse_port = !!(inp->inp_socket->so_options
                                             & SO_REUSEPORT);
                        vtw->reuse_addr = !!(inp->inp_socket->so_options
                                             & SO_REUSEADDR);
                        vtw->v6only     = 0;
                        vtw->uid        = inp->inp_socket->so_uidinfo->ui_uid;

                        vtw_inshash_v4(ctl, vtw);


#ifdef VTW_DEBUG
                        /* Immediate lookup (connected and port) to
                         * ensure at least that works!
                         */
                        if (enable & 4) {
                                KASSERT(vtw_lookup_hash_v4
                                        (ctl
                                         , in4p_faddr(inp).s_addr, inp->inp_fport
                                         , in4p_laddr(inp).s_addr, inp->inp_lport
                                         , 0)
                                        == vtw);
                                KASSERT(vtw_lookup_hash_v4
                                        (ctl
                                         , in4p_faddr(inp).s_addr, inp->inp_fport
                                         , in4p_laddr(inp).s_addr, inp->inp_lport
                                         , 1));
                        }
                        /* Immediate port iterator functionality check: not wild
                         */
                        if (enable & 8) {
                                struct tcp_ports_iterator *it;
                                struct vestigial_inpcb res;
                                int cnt = 0;

                                it = tcp_init_ports_v4(in4p_laddr(inp)
                                                       , inp->inp_lport, 0);

                                while (tcp_next_port_v4(it, &res)) {
                                        ++cnt;
                                }
                                KASSERT(cnt);
                        }
                        /* Immediate port iterator functionality check: wild
                         */
                        if (enable & 16) {
                                struct tcp_ports_iterator *it;
                                struct vestigial_inpcb res;
                                struct in_addr any;
                                int cnt = 0;

                                any.s_addr = htonl(INADDR_ANY);

                                it = tcp_init_ports_v4(any, inp->inp_lport, 1);

                                while (tcp_next_port_v4(it, &res)) {
                                        ++cnt;
                                }
                                KASSERT(cnt);
                        }
#endif /* VTW_DEBUG */
                        break;
                }

                case AF_INET6: {
                        struct inpcb    *inp = tp->t_inpcb;
                        vtw_v6_t        *v6  = (void*)vtw;

                        v6->faddr = in6p_faddr(inp);
                        v6->laddr = in6p_laddr(inp);
                        v6->fport = inp->inp_fport;
                        v6->lport = inp->inp_lport;

                        vtw->reuse_port = !!(inp->inp_socket->so_options
                                             & SO_REUSEPORT);
                        vtw->reuse_addr = !!(inp->inp_socket->so_options
                                             & SO_REUSEADDR);
                        vtw->v6only     = !!(inp->inp_flags
                                             & IN6P_IPV6_V6ONLY);
                        vtw->uid        = inp->inp_socket->so_uidinfo->ui_uid;

                        vtw_inshash_v6(ctl, vtw);
#ifdef VTW_DEBUG
                        /* Immediate lookup (connected and port) to
                         * ensure at least that works!
                         */
                        if (enable & 4) {
                                KASSERT(vtw_lookup_hash_v6(ctl
                                         , &in6p_faddr(inp), inp->inp_fport
                                         , &in6p_laddr(inp), inp->inp_lport
                                         , 0)
                                        == vtw);
                                KASSERT(vtw_lookup_hash_v6
                                        (ctl
                                         , &in6p_faddr(inp), inp->inp_fport
                                         , &in6p_laddr(inp), inp->inp_lport
                                         , 1));
                        }
                        /* Immediate port iterator functionality check: not wild
                         */
                        if (enable & 8) {
                                struct tcp_ports_iterator *it;
                                struct vestigial_inpcb res;
                                int cnt = 0;

                                it = tcp_init_ports_v6(&in6p_laddr(inp)
                                                       , inp->inp_lport, 0);

                                while (tcp_next_port_v6(it, &res)) {
                                        ++cnt;
                                }
                                KASSERT(cnt);
                        }
                        /* Immediate port iterator functionality check: wild
                         */
                        if (enable & 16) {
                                struct tcp_ports_iterator *it;
                                struct vestigial_inpcb res;
                                static struct in6_addr any = IN6ADDR_ANY_INIT;
                                int cnt = 0;

                                it = tcp_init_ports_v6(&any
                                                       , inp->inp_lport, 1);

                                while (tcp_next_port_v6(it, &res)) {
                                        ++cnt;
                                }
                                KASSERT(cnt);
                        }
#endif /* VTW_DEBUG */
                        break;
                }
                }

                tcp_canceltimers(tp);
                tp = tcp_close(tp);
                KASSERT(!tp);

                return 1;
        }

        return 0;
}

/*!\brief       restart timer for vestigial time-wait entry
 */
static void
vtw_restart_v4(vestigial_inpcb_t *vp)
{
        vtw_v4_t        copy = *(vtw_v4_t*)vp->vtw;
        vtw_t           *vtw;
        vtw_t           *cp  = &copy.common;
        vtw_ctl_t       *ctl;

        KASSERT(mutex_owned(softnet_lock));

        db_trace(KTR_VTW
                 , (vp->vtw, "vtw: restart %A:%P %A:%P"
                    , vp->faddr.v4.s_addr, vp->fport
                    , vp->laddr.v4.s_addr, vp->lport));

        /* Class might have changed, so have a squiz.
         */
        ctl = vtw_control(AF_INET, class_to_msl(cp->msl_class));
        vtw = vtw_alloc(ctl);

        if (vtw) {
                vtw_v4_t        *v4  = (void*)vtw;

                /* Safe now to unhash the old entry
                 */
                vtw_del(vp->ctl, vp->vtw);

                vtw->snd_nxt = cp->snd_nxt;
                vtw->rcv_nxt = cp->rcv_nxt;

                v4->faddr = copy.faddr;
                v4->laddr = copy.laddr;
                v4->fport = copy.fport;
                v4->lport = copy.lport;

                vtw->reuse_port = cp->reuse_port;
                vtw->reuse_addr = cp->reuse_addr;
                vtw->v6only     = 0;
                vtw->uid        = cp->uid;

                vtw_inshash_v4(ctl, vtw);
        }

        vp->valid = 0;
}

/*!\brief       restart timer for vestigial time-wait entry
 */
static void
vtw_restart_v6(vestigial_inpcb_t *vp)
{
        vtw_v6_t        copy = *(vtw_v6_t*)vp->vtw;
        vtw_t           *vtw;
        vtw_t           *cp  = &copy.common;
        vtw_ctl_t       *ctl;

        KASSERT(mutex_owned(softnet_lock));

        db_trace(KTR_VTW
                 , (vp->vtw, "vtw: restart %6A:%P %6A:%P"
                    , db_store(&vp->faddr.v6, sizeof (vp->faddr.v6))
                    , vp->fport
                    , db_store(&vp->laddr.v6, sizeof (vp->laddr.v6))
                    , vp->lport));

        /* Class might have changed, so have a squiz.
         */
        ctl = vtw_control(AF_INET6, class_to_msl(cp->msl_class));
        vtw = vtw_alloc(ctl);

        if (vtw) {
                vtw_v6_t        *v6  = (void*)vtw;

                /* Safe now to unhash the old entry
                 */
                vtw_del(vp->ctl, vp->vtw);

                vtw->snd_nxt = cp->snd_nxt;
                vtw->rcv_nxt = cp->rcv_nxt;

                v6->faddr = copy.faddr;
                v6->laddr = copy.laddr;
                v6->fport = copy.fport;
                v6->lport = copy.lport;

                vtw->reuse_port = cp->reuse_port;
                vtw->reuse_addr = cp->reuse_addr;
                vtw->v6only     = cp->v6only;
                vtw->uid        = cp->uid;

                vtw_inshash_v6(ctl, vtw);
        }

        vp->valid = 0;
}

/*!\brief       restart timer for vestigial time-wait entry
 */
void
vtw_restart(vestigial_inpcb_t *vp)
{
        if (!vp || !vp->valid)
                return;

        if (vp->v4)
                vtw_restart_v4(vp);
        else
                vtw_restart_v6(vp);
}

int
sysctl_tcp_vtw_enable(SYSCTLFN_ARGS)
{  
        int en, rc;
        struct sysctlnode node;

        node = *rnode;
        en = *(int *)rnode->sysctl_data;
        node.sysctl_data = &en;

        rc = sysctl_lookup(SYSCTLFN_CALL(&node));
        if (rc != 0 || newp == NULL)
                return rc;

        if (rnode->sysctl_data != &tcp4_vtw_enable &&
            rnode->sysctl_data != &tcp6_vtw_enable)
                rc = ENOENT;
        else if ((en & 1) == 0)
                rc = 0;
        else if (rnode->sysctl_data == &tcp4_vtw_enable)
                rc = vtw_control_init(AF_INET);
        else /* rnode->sysctl_data == &tcp6_vtw_enable */
                rc = vtw_control_init(AF_INET6);

        if (rc == 0)
                *(int *)rnode->sysctl_data = en;

        return rc;
}

int
vtw_earlyinit(void)
{
        int i, rc;

        callout_init(&vtw_cs, 0);
        callout_setfunc(&vtw_cs, vtw_tick, 0);

        for (i = 0; i < VTW_NCLASS; ++i) {
                vtw_tcpv4[i].is_v4 = 1;
                vtw_tcpv6[i].is_v6 = 1;
        }

        if ((tcp4_vtw_enable & 1) != 0 &&
            (rc = vtw_control_init(AF_INET)) != 0)
                return rc;

        if ((tcp6_vtw_enable & 1) != 0 &&
            (rc = vtw_control_init(AF_INET6)) != 0)
                return rc;

        return 0;
}

#ifdef VTW_DEBUG
#include <sys/syscallargs.h>
#include <sys/sysctl.h>

/*!\brief       add lalp, fafp entries for debug
 */
int
vtw_debug_add(int af, sin_either_t *la, sin_either_t *fa, int msl, int msl_class)
{
        vtw_ctl_t       *ctl;
        vtw_t           *vtw;

        ctl = vtw_control(af, msl ? msl : class_to_msl(msl_class));
        if (!ctl)
                return 0;

        vtw = vtw_alloc(ctl);

        if (vtw) {
                vtw->snd_nxt = 0;
                vtw->rcv_nxt = 0;

                switch (af) {
                case AF_INET: {
                        vtw_v4_t        *v4  = (void*)vtw;

                        v4->faddr = fa->sin_addr.v4.s_addr;
                        v4->laddr = la->sin_addr.v4.s_addr;
                        v4->fport = fa->sin_port;
                        v4->lport = la->sin_port;

                        vtw->reuse_port = 1;
                        vtw->reuse_addr = 1;
                        vtw->v6only     = 0;
                        vtw->uid        = 0;

                        vtw_inshash_v4(ctl, vtw);
                        break;
                }

                case AF_INET6: {
                        vtw_v6_t        *v6  = (void*)vtw;

                        v6->faddr = fa->sin_addr.v6;
                        v6->laddr = la->sin_addr.v6;

                        v6->fport = fa->sin_port;
                        v6->lport = la->sin_port;

                        vtw->reuse_port = 1;
                        vtw->reuse_addr = 1;
                        vtw->v6only     = 0;
                        vtw->uid        = 0;

                        vtw_inshash_v6(ctl, vtw);
                        break;
                }

                default:
                        break;
                }

                return 1;
        }

        return 0;
}

static int vtw_syscall = 0;

static int
vtw_debug_process(vtw_sysargs_t *ap)
{
        struct vestigial_inpcb vestige;
        int     rc = 0;

        mutex_enter(softnet_lock);

        switch (ap->op) {
        case 0:         // insert
                vtw_debug_add(ap->la.sin_family
                              , &ap->la
                              , &ap->fa
                              , TCPTV_MSL
                              , 0);
                break;

        case 1:         // lookup
        case 2:         // restart
                switch (ap->la.sin_family) {
                case AF_INET:
                        if (tcp_lookup_v4(ap->fa.sin_addr.v4, ap->fa.sin_port,
                                          ap->la.sin_addr.v4, ap->la.sin_port,
                                          &vestige)) {
                                if (ap->op == 2) {
                                        vtw_restart(&vestige);
                                }
                                rc = 0;
                        } else
                                rc = ESRCH;
                        break;

                case AF_INET6:
                        if (tcp_lookup_v6(&ap->fa.sin_addr.v6, ap->fa.sin_port,
                                          &ap->la.sin_addr.v6, ap->la.sin_port,
                                          &vestige)) {
                                if (ap->op == 2) {
                                        vtw_restart(&vestige);
                                }
                                rc = 0;
                        } else
                                rc = ESRCH;
                        break;
                default:
                        rc = EINVAL;
                }
                break;

        default:
                rc = EINVAL;
        }

        mutex_exit(softnet_lock);
        return rc;
}

struct sys_vtw_args {
        syscallarg(const vtw_sysargs_t *) req;
        syscallarg(size_t) len;
};

static int
vtw_sys(struct lwp *l, const void *_, register_t *retval)
{
        const struct sys_vtw_args *uap = _;
        void    *buf;
        int     rc;
        size_t  len     = SCARG(uap, len);

        if (len != sizeof (vtw_sysargs_t))
                return EINVAL;

        buf = kmem_alloc(len, KM_SLEEP);
        rc = copyin(SCARG(uap, req), buf, len);
        if (!rc) {
                rc = vtw_debug_process(buf);
        }
        kmem_free(buf, len);

        return rc;
}

static void
vtw_sanity_check(void)
{
        vtw_ctl_t       *ctl;
        vtw_t           *vtw;
        int             i;
        int             n;

        for (i = 0; i < VTW_NCLASS; ++i) {
                ctl = &vtw_tcpv4[i];

                if (!ctl->base.v || ctl->nalloc)
                        continue;

                for (n = 0, vtw = ctl->base.v; ; ) {
                        ++n;
                        vtw = vtw_next(ctl, vtw);
                        if (vtw == ctl->base.v)
                                break;
                }
                db_trace(KTR_VTW
                         , (ctl, "sanity: class %x n %x nfree %x"
                            , i, n, ctl->nfree));

                KASSERT(n == ctl->nfree);
        }

        for (i = 0; i < VTW_NCLASS; ++i) {
                ctl = &vtw_tcpv6[i];

                if (!ctl->base.v || ctl->nalloc)
                        continue;

                for (n = 0, vtw = ctl->base.v; ; ) {
                        ++n;
                        vtw = vtw_next(ctl, vtw);
                        if (vtw == ctl->base.v)
                                break;
                }
                db_trace(KTR_VTW
                         , (ctl, "sanity: class %x n %x nfree %x"
                            , i, n, ctl->nfree));
                KASSERT(n == ctl->nfree);
        }
}
                
/*!\brief       Initialise debug support.
 */
static void
vtw_debug_init(void)
{
        int     i;

        vtw_sanity_check();

        if (vtw_syscall)
                return;

        for (i = 511; i; --i) {
                if (sysent[i].sy_call == sys_nosys) {
                        sysent[i].sy_call    = vtw_sys;
                        sysent[i].sy_narg    = 2;
                        sysent[i].sy_argsize = sizeof (struct sys_vtw_args);
                        sysent[i].sy_flags   = 0;

                        vtw_syscall = i;
                        break;
                }
        }
        if (i) {
                const struct sysctlnode *node;
                uint32_t        flags;

                flags = sysctl_root.sysctl_flags;

                sysctl_root.sysctl_flags |= CTLFLAG_READWRITE;
                sysctl_root.sysctl_flags &= ~CTLFLAG_PERMANENT;

                sysctl_createv(0, 0, 0, &node,
                               CTLFLAG_PERMANENT, CTLTYPE_NODE,
                               "koff",
                               SYSCTL_DESCR("Kernel Obscure Feature Finder"),
                               0, 0, 0, 0, CTL_CREATE, CTL_EOL);

                if (!node) {
                        sysctl_createv(0, 0, 0, &node,
                                       CTLFLAG_PERMANENT, CTLTYPE_NODE,
                                       "koffka",
                                       SYSCTL_DESCR("The Real(tm) Kernel"
                                                    " Obscure Feature Finder"),
                                       0, 0, 0, 0, CTL_CREATE, CTL_EOL);
                }
                if (node) {
                        sysctl_createv(0, 0, 0, 0,
                                       CTLFLAG_PERMANENT|CTLFLAG_READONLY,
                                       CTLTYPE_INT, "vtw_debug_syscall",
                                       SYSCTL_DESCR("vtw debug"
                                                    " system call number"),
                                       0, 0, &vtw_syscall, 0, node->sysctl_num,
                                       CTL_CREATE, CTL_EOL);
                }
                sysctl_root.sysctl_flags = flags;
        }
}
#else /* !VTW_DEBUG */
static void
vtw_debug_init(void)
{
        return;
}
#endif /* !VTW_DEBUG */