/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */

/*
 * Copyright 2019 Joyent, Inc.
 * Copyright (c) 2013, 2015 by Delphix. All rights reserved.
 */

#include "umem.h"

#include <sys/vmem_impl_user.h>
#include <umem_impl.h>

#include <alloca.h>
#include <limits.h>
#include <mdb/mdb_whatis.h>
#include <thr_uberdata.h>

#include "misc.h"
#include "leaky.h"
#include "dist.h"

#include "umem_pagesize.h"

#define UM_ALLOCATED            0x1
#define UM_FREE                 0x2
#define UM_BUFCTL               0x4
#define UM_HASH                 0x8

int umem_ready;

static int umem_stack_depth_warned;
static uint32_t umem_max_ncpus;
uint32_t umem_stack_depth;

size_t umem_pagesize;

#define UMEM_READVAR(var)                               \
        (umem_readvar(&(var), #var) == -1 &&            \
            (mdb_warn("failed to read "#var), 1))

int
umem_update_variables(void)
{
        size_t pagesize;

        /*
         * Figure out which type of umem is being used; if it's not there
         * yet, succeed quietly.
         */
        if (umem_set_standalone() == -1) {
                umem_ready = 0;
                return (0);             /* umem not there yet */
        }

        /*
         * Solaris 9 used a different name for umem_max_ncpus.  It's
         * cheap backwards compatibility to check for both names.
         */
        if (umem_readvar(&umem_max_ncpus, "umem_max_ncpus") == -1 &&
            umem_readvar(&umem_max_ncpus, "max_ncpus") == -1) {
                mdb_warn("unable to read umem_max_ncpus or max_ncpus");
                return (-1);
        }
        if (UMEM_READVAR(umem_ready))
                return (-1);
        if (UMEM_READVAR(umem_stack_depth))
                return (-1);
        if (UMEM_READVAR(pagesize))
                return (-1);

        if (umem_stack_depth > UMEM_MAX_STACK_DEPTH) {
                if (umem_stack_depth_warned == 0) {
                        mdb_warn("umem_stack_depth corrupted (%d > %d)\n",
                            umem_stack_depth, UMEM_MAX_STACK_DEPTH);
                        umem_stack_depth_warned = 1;
                }
                umem_stack_depth = 0;
        }

        umem_pagesize = pagesize;

        return (0);
}

static int
umem_ptc_walk_init(mdb_walk_state_t *wsp)
{
        if (wsp->walk_addr == 0) {
                if (mdb_layered_walk("ulwp", wsp) == -1) {
                        mdb_warn("couldn't walk 'ulwp'");
                        return (WALK_ERR);
                }
        }

        return (WALK_NEXT);
}

static int
umem_ptc_walk_step(mdb_walk_state_t *wsp)
{
        uintptr_t this;
        int rval;

        if (wsp->walk_layer != NULL) {
                this = (uintptr_t)((ulwp_t *)wsp->walk_layer)->ul_self +
                    (uintptr_t)wsp->walk_arg;
        } else {
                this = wsp->walk_addr + (uintptr_t)wsp->walk_arg;
        }

        for (;;) {
                if (mdb_vread(&this, sizeof (void *), this) == -1) {
                        mdb_warn("couldn't read ptc buffer at %p", this);
                        return (WALK_ERR);
                }

                if (this == 0)
                        break;

                rval = wsp->walk_callback(this, &this, wsp->walk_cbdata);

                if (rval != WALK_NEXT)
                        return (rval);
        }

        return (wsp->walk_layer != NULL ? WALK_NEXT : WALK_DONE);
}

/*ARGSUSED*/
static int
umem_init_walkers(uintptr_t addr, const umem_cache_t *c, int *sizes)
{
        mdb_walker_t w;
        char descr[64];
        char name[64];
        int i;

        (void) mdb_snprintf(descr, sizeof (descr),
            "walk the %s cache", c->cache_name);

        w.walk_name = c->cache_name;
        w.walk_descr = descr;
        w.walk_init = umem_walk_init;
        w.walk_step = umem_walk_step;
        w.walk_fini = umem_walk_fini;
        w.walk_init_arg = (void *)addr;

        if (mdb_add_walker(&w) == -1)
                mdb_warn("failed to add %s walker", c->cache_name);

        if (!(c->cache_flags & UMF_PTC))
                return (WALK_NEXT);

        /*
         * For the per-thread cache walker, the address is the offset in the
         * tm_roots[] array of the ulwp_t.
         */
        for (i = 0; sizes[i] != 0; i++) {
                if (sizes[i] == c->cache_bufsize)
                        break;
        }

        if (sizes[i] == 0) {
                mdb_warn("cache %s is cached per-thread, but could not find "
                    "size in umem_alloc_sizes\n", c->cache_name);
                return (WALK_NEXT);
        }

        if (i >= NTMEMBASE) {
                mdb_warn("index for %s (%d) exceeds root slots (%d)\n",
                    c->cache_name, i, NTMEMBASE);
                return (WALK_NEXT);
        }

        (void) mdb_snprintf(name, sizeof (name),
            "umem_ptc_%d", c->cache_bufsize);
        (void) mdb_snprintf(descr, sizeof (descr),
            "walk the per-thread cache for %s", c->cache_name);

        w.walk_name = name;
        w.walk_descr = descr;
        w.walk_init = umem_ptc_walk_init;
        w.walk_step = umem_ptc_walk_step;
        w.walk_fini = NULL;
        w.walk_init_arg = (void *)offsetof(ulwp_t, ul_tmem.tm_roots[i]);

        if (mdb_add_walker(&w) == -1)
                mdb_warn("failed to add %s walker", w.walk_name);

        return (WALK_NEXT);
}

/*ARGSUSED*/
static void
umem_statechange_cb(void *arg)
{
        static int been_ready = 0;
        GElf_Sym sym;
        int *sizes;

#ifndef _KMDB
        leaky_cleanup(1);       /* state changes invalidate leaky state */
#endif

        if (umem_update_variables() == -1)
                return;

        if (been_ready)
                return;

        if (umem_ready != UMEM_READY)
                return;

        been_ready = 1;

        /*
         * In order to determine the tm_roots offset of any cache that is
         * cached per-thread, we need to have the umem_alloc_sizes array.
         * Read this, assuring that it is zero-terminated.
         */
        if (umem_lookup_by_name("umem_alloc_sizes", &sym) == -1) {
                mdb_warn("unable to lookup 'umem_alloc_sizes'");
                return;
        }

        sizes = mdb_zalloc(sym.st_size + sizeof (int), UM_SLEEP | UM_GC);

        if (mdb_vread(sizes, sym.st_size, (uintptr_t)sym.st_value) == -1) {
                mdb_warn("couldn't read 'umem_alloc_sizes'");
                return;
        }

        (void) mdb_walk("umem_cache", (mdb_walk_cb_t)umem_init_walkers, sizes);
}

int
umem_abort_messages(void)
{
        char *umem_error_buffer;
        uint_t umem_error_begin;
        GElf_Sym sym;
        size_t bufsize;

        if (UMEM_READVAR(umem_error_begin))
                return (DCMD_ERR);

        if (umem_lookup_by_name("umem_error_buffer", &sym) == -1) {
                mdb_warn("unable to look up umem_error_buffer");
                return (DCMD_ERR);
        }

        bufsize = (size_t)sym.st_size;

        umem_error_buffer = mdb_alloc(bufsize+1, UM_SLEEP | UM_GC);

        if (mdb_vread(umem_error_buffer, bufsize, (uintptr_t)sym.st_value)
            != bufsize) {
                mdb_warn("unable to read umem_error_buffer");
                return (DCMD_ERR);
        }
        /* put a zero after the end of the buffer to simplify printing */
        umem_error_buffer[bufsize] = 0;

        if ((umem_error_begin % bufsize) == 0)
                mdb_printf("%s\n", umem_error_buffer);
        else {
                umem_error_buffer[(umem_error_begin % bufsize) - 1] = 0;
                mdb_printf("%s%s\n",
                    &umem_error_buffer[umem_error_begin % bufsize],
                    umem_error_buffer);
        }

        return (DCMD_OK);
}

static void
umem_log_status(const char *name, umem_log_header_t *val)
{
        umem_log_header_t my_lh;
        uintptr_t pos = (uintptr_t)val;
        size_t size;

        if (pos == 0)
                return;

        if (mdb_vread(&my_lh, sizeof (umem_log_header_t), pos) == -1) {
                mdb_warn("\nunable to read umem_%s_log pointer %p",
                    name, pos);
                return;
        }

        size = my_lh.lh_chunksize * my_lh.lh_nchunks;

        if (size % (1024 * 1024) == 0)
                mdb_printf("%s=%dm ", name, size / (1024 * 1024));
        else if (size % 1024 == 0)
                mdb_printf("%s=%dk ", name, size / 1024);
        else
                mdb_printf("%s=%d ", name, size);
}

typedef struct umem_debug_flags {
        const char      *udf_name;
        uint_t          udf_flags;
        uint_t          udf_clear;      /* if 0, uses udf_flags */
} umem_debug_flags_t;

umem_debug_flags_t umem_status_flags[] = {
        { "random",     UMF_RANDOMIZE,  UMF_RANDOM },
        { "default",    UMF_AUDIT | UMF_DEADBEEF | UMF_REDZONE | UMF_CONTENTS },
        { "audit",      UMF_AUDIT },
        { "guards",     UMF_DEADBEEF | UMF_REDZONE },
        { "nosignal",   UMF_CHECKSIGNAL },
        { "firewall",   UMF_FIREWALL },
        { "lite",       UMF_LITE },
        { "checknull",  UMF_CHECKNULL },
        { NULL }
};

/*ARGSUSED*/
int
umem_status(uintptr_t addr, uint_t flags, int ac, const mdb_arg_t *argv)
{
        int umem_logging;

        umem_log_header_t *umem_transaction_log;
        umem_log_header_t *umem_content_log;
        umem_log_header_t *umem_failure_log;
        umem_log_header_t *umem_slab_log;

        mdb_printf("Status:\t\t%s\n",
            umem_ready == UMEM_READY_INIT_FAILED ? "initialization failed" :
            umem_ready == UMEM_READY_STARTUP ? "uninitialized" :
            umem_ready == UMEM_READY_INITING ? "initialization in process" :
            umem_ready == UMEM_READY ? "ready and active" :
            umem_ready == 0 ? "not loaded into address space" :
            "unknown (umem_ready invalid)");

        if (umem_ready == 0)
                return (DCMD_OK);

        mdb_printf("Concurrency:\t%d\n", umem_max_ncpus);

        if (UMEM_READVAR(umem_logging))
                goto err;
        if (UMEM_READVAR(umem_transaction_log))
                goto err;
        if (UMEM_READVAR(umem_content_log))
                goto err;
        if (UMEM_READVAR(umem_failure_log))
                goto err;
        if (UMEM_READVAR(umem_slab_log))
                goto err;

        mdb_printf("Logs:\t\t");
        umem_log_status("transaction", umem_transaction_log);
        umem_log_status("content", umem_content_log);
        umem_log_status("fail", umem_failure_log);
        umem_log_status("slab", umem_slab_log);
        if (!umem_logging)
                mdb_printf("(inactive)");
        mdb_printf("\n");

        mdb_printf("Message buffer:\n");
        return (umem_abort_messages());

err:
        mdb_printf("Message buffer:\n");
        (void) umem_abort_messages();
        return (DCMD_ERR);
}

typedef struct {
        uintptr_t ucw_first;
        uintptr_t ucw_current;
} umem_cache_walk_t;

int
umem_cache_walk_init(mdb_walk_state_t *wsp)
{
        umem_cache_walk_t *ucw;
        umem_cache_t c;
        uintptr_t cp;
        GElf_Sym sym;

        if (umem_lookup_by_name("umem_null_cache", &sym) == -1) {
                mdb_warn("couldn't find umem_null_cache");
                return (WALK_ERR);
        }

        cp = (uintptr_t)sym.st_value;

        if (mdb_vread(&c, sizeof (umem_cache_t), cp) == -1) {
                mdb_warn("couldn't read cache at %p", cp);
                return (WALK_ERR);
        }

        ucw = mdb_alloc(sizeof (umem_cache_walk_t), UM_SLEEP);

        ucw->ucw_first = cp;
        ucw->ucw_current = (uintptr_t)c.cache_next;
        wsp->walk_data = ucw;

        return (WALK_NEXT);
}

int
umem_cache_walk_step(mdb_walk_state_t *wsp)
{
        umem_cache_walk_t *ucw = wsp->walk_data;
        umem_cache_t c;
        int status;

        if (mdb_vread(&c, sizeof (umem_cache_t), ucw->ucw_current) == -1) {
                mdb_warn("couldn't read cache at %p", ucw->ucw_current);
                return (WALK_DONE);
        }

        status = wsp->walk_callback(ucw->ucw_current, &c, wsp->walk_cbdata);

        if ((ucw->ucw_current = (uintptr_t)c.cache_next) == ucw->ucw_first)
                return (WALK_DONE);

        return (status);
}

void
umem_cache_walk_fini(mdb_walk_state_t *wsp)
{
        umem_cache_walk_t *ucw = wsp->walk_data;
        mdb_free(ucw, sizeof (umem_cache_walk_t));
}

typedef struct {
        umem_cpu_t *ucw_cpus;
        uint32_t ucw_current;
        uint32_t ucw_max;
} umem_cpu_walk_state_t;

int
umem_cpu_walk_init(mdb_walk_state_t *wsp)
{
        umem_cpu_t *umem_cpus;

        umem_cpu_walk_state_t *ucw;

        if (umem_readvar(&umem_cpus, "umem_cpus") == -1) {
                mdb_warn("failed to read 'umem_cpus'");
                return (WALK_ERR);
        }

        ucw = mdb_alloc(sizeof (*ucw), UM_SLEEP);

        ucw->ucw_cpus = umem_cpus;
        ucw->ucw_current = 0;
        ucw->ucw_max = umem_max_ncpus;

        wsp->walk_data = ucw;
        return (WALK_NEXT);
}

int
umem_cpu_walk_step(mdb_walk_state_t *wsp)
{
        umem_cpu_t cpu;
        umem_cpu_walk_state_t *ucw = wsp->walk_data;

        uintptr_t caddr;

        if (ucw->ucw_current >= ucw->ucw_max)
                return (WALK_DONE);

        caddr = (uintptr_t)&(ucw->ucw_cpus[ucw->ucw_current]);

        if (mdb_vread(&cpu, sizeof (umem_cpu_t), caddr) == -1) {
                mdb_warn("failed to read cpu %d", ucw->ucw_current);
                return (WALK_ERR);
        }

        ucw->ucw_current++;

        return (wsp->walk_callback(caddr, &cpu, wsp->walk_cbdata));
}

void
umem_cpu_walk_fini(mdb_walk_state_t *wsp)
{
        umem_cpu_walk_state_t *ucw = wsp->walk_data;

        mdb_free(ucw, sizeof (*ucw));
}

int
umem_cpu_cache_walk_init(mdb_walk_state_t *wsp)
{
        if (wsp->walk_addr == 0) {
                mdb_warn("umem_cpu_cache doesn't support global walks");
                return (WALK_ERR);
        }

        if (mdb_layered_walk("umem_cpu", wsp) == -1) {
                mdb_warn("couldn't walk 'umem_cpu'");
                return (WALK_ERR);
        }

        wsp->walk_data = (void *)wsp->walk_addr;

        return (WALK_NEXT);
}

int
umem_cpu_cache_walk_step(mdb_walk_state_t *wsp)
{
        uintptr_t caddr = (uintptr_t)wsp->walk_data;
        const umem_cpu_t *cpu = wsp->walk_layer;
        umem_cpu_cache_t cc;

        caddr += cpu->cpu_cache_offset;

        if (mdb_vread(&cc, sizeof (umem_cpu_cache_t), caddr) == -1) {
                mdb_warn("couldn't read umem_cpu_cache at %p", caddr);
                return (WALK_ERR);
        }

        return (wsp->walk_callback(caddr, &cc, wsp->walk_cbdata));
}

int
umem_slab_walk_init(mdb_walk_state_t *wsp)
{
        uintptr_t caddr = wsp->walk_addr;
        umem_cache_t c;

        if (caddr == 0) {
                mdb_warn("umem_slab doesn't support global walks\n");
                return (WALK_ERR);
        }

        if (mdb_vread(&c, sizeof (c), caddr) == -1) {
                mdb_warn("couldn't read umem_cache at %p", caddr);
                return (WALK_ERR);
        }

        wsp->walk_data =
            (void *)(caddr + offsetof(umem_cache_t, cache_nullslab));
        wsp->walk_addr = (uintptr_t)c.cache_nullslab.slab_next;

        return (WALK_NEXT);
}

int
umem_slab_walk_partial_init(mdb_walk_state_t *wsp)
{
        uintptr_t caddr = wsp->walk_addr;
        umem_cache_t c;

        if (caddr == 0) {
                mdb_warn("umem_slab_partial doesn't support global walks\n");
                return (WALK_ERR);
        }

        if (mdb_vread(&c, sizeof (c), caddr) == -1) {
                mdb_warn("couldn't read umem_cache at %p", caddr);
                return (WALK_ERR);
        }

        wsp->walk_data =
            (void *)(caddr + offsetof(umem_cache_t, cache_nullslab));
        wsp->walk_addr = (uintptr_t)c.cache_freelist;

        /*
         * Some consumers (umem_walk_step(), in particular) require at
         * least one callback if there are any buffers in the cache.  So
         * if there are *no* partial slabs, report the last full slab, if
         * any.
         *
         * Yes, this is ugly, but it's cleaner than the other possibilities.
         */
        if ((uintptr_t)wsp->walk_data == wsp->walk_addr)
                wsp->walk_addr = (uintptr_t)c.cache_nullslab.slab_prev;

        return (WALK_NEXT);
}

int
umem_slab_walk_step(mdb_walk_state_t *wsp)
{
        umem_slab_t s;
        uintptr_t addr = wsp->walk_addr;
        uintptr_t saddr = (uintptr_t)wsp->walk_data;
        uintptr_t caddr = saddr - offsetof(umem_cache_t, cache_nullslab);

        if (addr == saddr)
                return (WALK_DONE);

        if (mdb_vread(&s, sizeof (s), addr) == -1) {
                mdb_warn("failed to read slab at %p", wsp->walk_addr);
                return (WALK_ERR);
        }

        if ((uintptr_t)s.slab_cache != caddr) {
                mdb_warn("slab %p isn't in cache %p (in cache %p)\n",
                    addr, caddr, s.slab_cache);
                return (WALK_ERR);
        }

        wsp->walk_addr = (uintptr_t)s.slab_next;

        return (wsp->walk_callback(addr, &s, wsp->walk_cbdata));
}

int
umem_cache(uintptr_t addr, uint_t flags, int ac, const mdb_arg_t *argv)
{
        umem_cache_t c;

        if (!(flags & DCMD_ADDRSPEC)) {
                if (mdb_walk_dcmd("umem_cache", "umem_cache", ac, argv) == -1) {
                        mdb_warn("can't walk umem_cache");
                        return (DCMD_ERR);
                }
                return (DCMD_OK);
        }

        if (DCMD_HDRSPEC(flags))
                mdb_printf("%-?s %-25s %4s %8s %8s %8s\n", "ADDR", "NAME",
                    "FLAG", "CFLAG", "BUFSIZE", "BUFTOTL");

        if (mdb_vread(&c, sizeof (c), addr) == -1) {
                mdb_warn("couldn't read umem_cache at %p", addr);
                return (DCMD_ERR);
        }

        mdb_printf("%0?p %-25s %04x %08x %8ld %8lld\n", addr, c.cache_name,
            c.cache_flags, c.cache_cflags, c.cache_bufsize, c.cache_buftotal);

        return (DCMD_OK);
}

static int
addrcmp(const void *lhs, const void *rhs)
{
        uintptr_t p1 = *((uintptr_t *)lhs);
        uintptr_t p2 = *((uintptr_t *)rhs);

        if (p1 < p2)
                return (-1);
        if (p1 > p2)
                return (1);
        return (0);
}

static int
bufctlcmp(const umem_bufctl_audit_t **lhs, const umem_bufctl_audit_t **rhs)
{
        const umem_bufctl_audit_t *bcp1 = *lhs;
        const umem_bufctl_audit_t *bcp2 = *rhs;

        if (bcp1->bc_timestamp > bcp2->bc_timestamp)
                return (-1);

        if (bcp1->bc_timestamp < bcp2->bc_timestamp)
                return (1);

        return (0);
}

typedef struct umem_hash_walk {
        uintptr_t *umhw_table;
        size_t umhw_nelems;
        size_t umhw_pos;
        umem_bufctl_t umhw_cur;
} umem_hash_walk_t;

int
umem_hash_walk_init(mdb_walk_state_t *wsp)
{
        umem_hash_walk_t *umhw;
        uintptr_t *hash;
        umem_cache_t c;
        uintptr_t haddr, addr = wsp->walk_addr;
        size_t nelems;
        size_t hsize;

        if (addr == 0) {
                mdb_warn("umem_hash doesn't support global walks\n");
                return (WALK_ERR);
        }

        if (mdb_vread(&c, sizeof (c), addr) == -1) {
                mdb_warn("couldn't read cache at addr %p", addr);
                return (WALK_ERR);
        }

        if (!(c.cache_flags & UMF_HASH)) {
                mdb_warn("cache %p doesn't have a hash table\n", addr);
                return (WALK_DONE);             /* nothing to do */
        }

        umhw = mdb_zalloc(sizeof (umem_hash_walk_t), UM_SLEEP);
        umhw->umhw_cur.bc_next = NULL;
        umhw->umhw_pos = 0;

        umhw->umhw_nelems = nelems = c.cache_hash_mask + 1;
        hsize = nelems * sizeof (uintptr_t);
        haddr = (uintptr_t)c.cache_hash_table;

        umhw->umhw_table = hash = mdb_alloc(hsize, UM_SLEEP);
        if (mdb_vread(hash, hsize, haddr) == -1) {
                mdb_warn("failed to read hash table at %p", haddr);
                mdb_free(hash, hsize);
                mdb_free(umhw, sizeof (umem_hash_walk_t));
                return (WALK_ERR);
        }

        wsp->walk_data = umhw;

        return (WALK_NEXT);
}

int
umem_hash_walk_step(mdb_walk_state_t *wsp)
{
        umem_hash_walk_t *umhw = wsp->walk_data;
        uintptr_t addr = 0;

        if ((addr = (uintptr_t)umhw->umhw_cur.bc_next) == 0) {
                while (umhw->umhw_pos < umhw->umhw_nelems) {
                        if ((addr = umhw->umhw_table[umhw->umhw_pos++]) != 0)
                                break;
                }
        }
        if (addr == 0)
                return (WALK_DONE);

        if (mdb_vread(&umhw->umhw_cur, sizeof (umem_bufctl_t), addr) == -1) {
                mdb_warn("couldn't read umem_bufctl_t at addr %p", addr);
                return (WALK_ERR);
        }

        return (wsp->walk_callback(addr, &umhw->umhw_cur, wsp->walk_cbdata));
}

void
umem_hash_walk_fini(mdb_walk_state_t *wsp)
{
        umem_hash_walk_t *umhw = wsp->walk_data;

        if (umhw == NULL)
                return;

        mdb_free(umhw->umhw_table, umhw->umhw_nelems * sizeof (uintptr_t));
        mdb_free(umhw, sizeof (umem_hash_walk_t));
}

/*
 * Find the address of the bufctl structure for the address 'buf' in cache
 * 'cp', which is at address caddr, and place it in *out.
 */
static int
umem_hash_lookup(umem_cache_t *cp, uintptr_t caddr, void *buf, uintptr_t *out)
{
        uintptr_t bucket = (uintptr_t)UMEM_HASH(cp, buf);
        umem_bufctl_t *bcp;
        umem_bufctl_t bc;

        if (mdb_vread(&bcp, sizeof (umem_bufctl_t *), bucket) == -1) {
                mdb_warn("unable to read hash bucket for %p in cache %p",
                    buf, caddr);
                return (-1);
        }

        while (bcp != NULL) {
                if (mdb_vread(&bc, sizeof (umem_bufctl_t),
                    (uintptr_t)bcp) == -1) {
                        mdb_warn("unable to read bufctl at %p", bcp);
                        return (-1);
                }
                if (bc.bc_addr == buf) {
                        *out = (uintptr_t)bcp;
                        return (0);
                }
                bcp = bc.bc_next;
        }

        mdb_warn("unable to find bufctl for %p in cache %p\n", buf, caddr);
        return (-1);
}

int
umem_get_magsize(const umem_cache_t *cp)
{
        uintptr_t addr = (uintptr_t)cp->cache_magtype;
        GElf_Sym mt_sym;
        umem_magtype_t mt;
        int res;

        /*
         * if cpu 0 has a non-zero magsize, it must be correct.  caches
         * with UMF_NOMAGAZINE have disabled their magazine layers, so
         * it is okay to return 0 for them.
         */
        if ((res = cp->cache_cpu[0].cc_magsize) != 0 ||
            (cp->cache_flags & UMF_NOMAGAZINE))
                return (res);

        if (umem_lookup_by_name("umem_magtype", &mt_sym) == -1) {
                mdb_warn("unable to read 'umem_magtype'");
        } else if (addr < mt_sym.st_value ||
            addr + sizeof (mt) - 1 > mt_sym.st_value + mt_sym.st_size - 1 ||
            ((addr - mt_sym.st_value) % sizeof (mt)) != 0) {
                mdb_warn("cache '%s' has invalid magtype pointer (%p)\n",
                    cp->cache_name, addr);
                return (0);
        }
        if (mdb_vread(&mt, sizeof (mt), addr) == -1) {
                mdb_warn("unable to read magtype at %a", addr);
                return (0);
        }
        return (mt.mt_magsize);
}

/*ARGSUSED*/
static int
umem_estimate_slab(uintptr_t addr, const umem_slab_t *sp, size_t *est)
{
        *est -= (sp->slab_chunks - sp->slab_refcnt);

        return (WALK_NEXT);
}

/*
 * Returns an upper bound on the number of allocated buffers in a given
 * cache.
 */
size_t
umem_estimate_allocated(uintptr_t addr, const umem_cache_t *cp)
{
        int magsize;
        size_t cache_est;

        cache_est = cp->cache_buftotal;

        (void) mdb_pwalk("umem_slab_partial",
            (mdb_walk_cb_t)umem_estimate_slab, &cache_est, addr);

        if ((magsize = umem_get_magsize(cp)) != 0) {
                size_t mag_est = cp->cache_full.ml_total * magsize;

                if (cache_est >= mag_est) {
                        cache_est -= mag_est;
                } else {
                        mdb_warn("cache %p's magazine layer holds more buffers "
                            "than the slab layer.\n", addr);
                }
        }
        return (cache_est);
}

#define READMAG_ROUNDS(rounds) { \
        if (mdb_vread(mp, magbsize, (uintptr_t)ump) == -1) { \
                mdb_warn("couldn't read magazine at %p", ump); \
                goto fail; \
        } \
        for (i = 0; i < rounds; i++) { \
                maglist[magcnt++] = mp->mag_round[i]; \
                if (magcnt == magmax) { \
                        mdb_warn("%d magazines exceeds fudge factor\n", \
                            magcnt); \
                        goto fail; \
                } \
        } \
}

static int
umem_read_magazines(umem_cache_t *cp, uintptr_t addr,
    void ***maglistp, size_t *magcntp, size_t *magmaxp)
{
        umem_magazine_t *ump, *mp;
        void **maglist = NULL;
        int i, cpu;
        size_t magsize, magmax, magbsize;
        size_t magcnt = 0;

        /*
         * Read the magtype out of the cache, after verifying the pointer's
         * correctness.
         */
        magsize = umem_get_magsize(cp);
        if (magsize == 0) {
                *maglistp = NULL;
                *magcntp = 0;
                *magmaxp = 0;
                return (0);
        }

        /*
         * There are several places where we need to go buffer hunting:
         * the per-CPU loaded magazine, the per-CPU spare full magazine,
         * and the full magazine list in the depot.
         *
         * For an upper bound on the number of buffers in the magazine
         * layer, we have the number of magazines on the cache_full
         * list plus at most two magazines per CPU (the loaded and the
         * spare).  Toss in 100 magazines as a fudge factor in case this
         * is live (the number "100" comes from the same fudge factor in
         * crash(8)).
         */
        magmax = (cp->cache_full.ml_total + 2 * umem_max_ncpus + 100) * magsize;
        magbsize = offsetof(umem_magazine_t, mag_round[magsize]);

        if (magbsize >= PAGESIZE / 2) {
                mdb_warn("magazine size for cache %p unreasonable (%x)\n",
                    addr, magbsize);
                return (-1);
        }

        maglist = mdb_alloc(magmax * sizeof (void *), UM_SLEEP);
        mp = mdb_alloc(magbsize, UM_SLEEP);
        if (mp == NULL || maglist == NULL)
                goto fail;

        /*
         * First up: the magazines in the depot (i.e. on the cache_full list).
         */
        for (ump = cp->cache_full.ml_list; ump != NULL; ) {
                READMAG_ROUNDS(magsize);
                ump = mp->mag_next;

                if (ump == cp->cache_full.ml_list)
                        break; /* cache_full list loop detected */
        }

        dprintf(("cache_full list done\n"));

        /*
         * Now whip through the CPUs, snagging the loaded magazines
         * and full spares.
         */
        for (cpu = 0; cpu < umem_max_ncpus; cpu++) {
                umem_cpu_cache_t *ccp = &cp->cache_cpu[cpu];

                dprintf(("reading cpu cache %p\n",
                    (uintptr_t)ccp - (uintptr_t)cp + addr));

                if (ccp->cc_rounds > 0 &&
                    (ump = ccp->cc_loaded) != NULL) {
                        dprintf(("reading %d loaded rounds\n", ccp->cc_rounds));
                        READMAG_ROUNDS(ccp->cc_rounds);
                }

                if (ccp->cc_prounds > 0 &&
                    (ump = ccp->cc_ploaded) != NULL) {
                        dprintf(("reading %d previously loaded rounds\n",
                            ccp->cc_prounds));
                        READMAG_ROUNDS(ccp->cc_prounds);
                }
        }

        dprintf(("magazine layer: %d buffers\n", magcnt));

        mdb_free(mp, magbsize);

        *maglistp = maglist;
        *magcntp = magcnt;
        *magmaxp = magmax;

        return (0);

fail:
        if (mp)
                mdb_free(mp, magbsize);
        if (maglist)
                mdb_free(maglist, magmax * sizeof (void *));

        return (-1);
}

typedef struct umem_read_ptc_walk {
        void **urpw_buf;
        size_t urpw_cnt;
        size_t urpw_max;
} umem_read_ptc_walk_t;

/*ARGSUSED*/
static int
umem_read_ptc_walk_buf(uintptr_t addr,
    const void *ignored, umem_read_ptc_walk_t *urpw)
{
        if (urpw->urpw_cnt == urpw->urpw_max) {
                size_t nmax = urpw->urpw_max ? (urpw->urpw_max << 1) : 1;
                void **new = mdb_zalloc(nmax * sizeof (void *), UM_SLEEP);

                if (nmax > 1) {
                        size_t osize = urpw->urpw_max * sizeof (void *);
                        bcopy(urpw->urpw_buf, new, osize);
                        mdb_free(urpw->urpw_buf, osize);
                }

                urpw->urpw_buf = new;
                urpw->urpw_max = nmax;
        }

        urpw->urpw_buf[urpw->urpw_cnt++] = (void *)addr;

        return (WALK_NEXT);
}

static int
umem_read_ptc(umem_cache_t *cp,
    void ***buflistp, size_t *bufcntp, size_t *bufmaxp)
{
        umem_read_ptc_walk_t urpw;
        char walk[60];
        int rval;

        if (!(cp->cache_flags & UMF_PTC))
                return (0);

        (void) mdb_snprintf(walk, sizeof (walk), "umem_ptc_%d",
            cp->cache_bufsize);

        urpw.urpw_buf = *buflistp;
        urpw.urpw_cnt = *bufcntp;
        urpw.urpw_max = *bufmaxp;

        if ((rval = mdb_walk(walk,
            (mdb_walk_cb_t)umem_read_ptc_walk_buf, &urpw)) == -1) {
                mdb_warn("couldn't walk %s", walk);
        }

        *buflistp = urpw.urpw_buf;
        *bufcntp = urpw.urpw_cnt;
        *bufmaxp = urpw.urpw_max;

        return (rval);
}

static int
umem_walk_callback(mdb_walk_state_t *wsp, uintptr_t buf)
{
        return (wsp->walk_callback(buf, NULL, wsp->walk_cbdata));
}

static int
bufctl_walk_callback(umem_cache_t *cp, mdb_walk_state_t *wsp, uintptr_t buf)
{
        umem_bufctl_audit_t *b;
        UMEM_LOCAL_BUFCTL_AUDIT(&b);

        /*
         * if UMF_AUDIT is not set, we know that we're looking at a
         * umem_bufctl_t.
         */
        if (!(cp->cache_flags & UMF_AUDIT) ||
            mdb_vread(b, UMEM_BUFCTL_AUDIT_SIZE, buf) == -1) {
                (void) memset(b, 0, UMEM_BUFCTL_AUDIT_SIZE);
                if (mdb_vread(b, sizeof (umem_bufctl_t), buf) == -1) {
                        mdb_warn("unable to read bufctl at %p", buf);
                        return (WALK_ERR);
                }
        }

        return (wsp->walk_callback(buf, b, wsp->walk_cbdata));
}

typedef struct umem_walk {
        int umw_type;

        uintptr_t umw_addr;             /* cache address */
        umem_cache_t *umw_cp;
        size_t umw_csize;

        /*
         * magazine layer
         */
        void **umw_maglist;
        size_t umw_max;
        size_t umw_count;
        size_t umw_pos;

        /*
         * slab layer
         */
        char *umw_valid;        /* to keep track of freed buffers */
        char *umw_ubase;        /* buffer for slab data */
} umem_walk_t;

static int
umem_walk_init_common(mdb_walk_state_t *wsp, int type)
{
        umem_walk_t *umw;
        int csize;
        umem_cache_t *cp;
        size_t vm_quantum;

        size_t magmax, magcnt;
        void **maglist = NULL;
        uint_t chunksize = 1, slabsize = 1;
        int status = WALK_ERR;
        uintptr_t addr = wsp->walk_addr;
        const char *layered;

        type &= ~UM_HASH;

        if (addr == 0) {
                mdb_warn("umem walk doesn't support global walks\n");
                return (WALK_ERR);
        }

        dprintf(("walking %p\n", addr));

        /*
         * The number of "cpus" determines how large the cache is.
         */
        csize = UMEM_CACHE_SIZE(umem_max_ncpus);
        cp = mdb_alloc(csize, UM_SLEEP);

        if (mdb_vread(cp, csize, addr) == -1) {
                mdb_warn("couldn't read cache at addr %p", addr);
                goto out2;
        }

        /*
         * It's easy for someone to hand us an invalid cache address.
         * Unfortunately, it is hard for this walker to survive an
         * invalid cache cleanly.  So we make sure that:
         *
         *      1. the vmem arena for the cache is readable,
         *      2. the vmem arena's quantum is a power of 2,
         *      3. our slabsize is a multiple of the quantum, and
         *      4. our chunksize is >0 and less than our slabsize.
         */
        if (mdb_vread(&vm_quantum, sizeof (vm_quantum),
            (uintptr_t)&cp->cache_arena->vm_quantum) == -1 ||
            vm_quantum == 0 ||
            (vm_quantum & (vm_quantum - 1)) != 0 ||
            cp->cache_slabsize < vm_quantum ||
            P2PHASE(cp->cache_slabsize, vm_quantum) != 0 ||
            cp->cache_chunksize == 0 ||
            cp->cache_chunksize > cp->cache_slabsize) {
                mdb_warn("%p is not a valid umem_cache_t\n", addr);
                goto out2;
        }

        dprintf(("buf total is %d\n", cp->cache_buftotal));

        if (cp->cache_buftotal == 0) {
                mdb_free(cp, csize);
                return (WALK_DONE);
        }

        /*
         * If they ask for bufctls, but it's a small-slab cache,
         * there is nothing to report.
         */
        if ((type & UM_BUFCTL) && !(cp->cache_flags & UMF_HASH)) {
                dprintf(("bufctl requested, not UMF_HASH (flags: %p)\n",
                    cp->cache_flags));
                mdb_free(cp, csize);
                return (WALK_DONE);
        }

        /*
         * Read in the contents of the magazine layer
         */
        if (umem_read_magazines(cp, addr, &maglist, &magcnt, &magmax) != 0)
                goto out2;

        /*
         * Read in the contents of the per-thread caches, if any
         */
        if (umem_read_ptc(cp, &maglist, &magcnt, &magmax) != 0)
                goto out2;

        /*
         * We have all of the buffers from the magazines and from the
         * per-thread cache (if any);  if we are walking allocated buffers,
         * sort them so we can bsearch them later.
         */
        if (type & UM_ALLOCATED)
                qsort(maglist, magcnt, sizeof (void *), addrcmp);

        wsp->walk_data = umw = mdb_zalloc(sizeof (umem_walk_t), UM_SLEEP);

        umw->umw_type = type;
        umw->umw_addr = addr;
        umw->umw_cp = cp;
        umw->umw_csize = csize;
        umw->umw_maglist = maglist;
        umw->umw_max = magmax;
        umw->umw_count = magcnt;
        umw->umw_pos = 0;

        /*
         * When walking allocated buffers in a UMF_HASH cache, we walk the
         * hash table instead of the slab layer.
         */
        if ((cp->cache_flags & UMF_HASH) && (type & UM_ALLOCATED)) {
                layered = "umem_hash";

                umw->umw_type |= UM_HASH;
        } else {
                /*
                 * If we are walking freed buffers, we only need the
                 * magazine layer plus the partially allocated slabs.
                 * To walk allocated buffers, we need all of the slabs.
                 */
                if (type & UM_ALLOCATED)
                        layered = "umem_slab";
                else
                        layered = "umem_slab_partial";

                /*
                 * for small-slab caches, we read in the entire slab.  For
                 * freed buffers, we can just walk the freelist.  For
                 * allocated buffers, we use a 'valid' array to track
                 * the freed buffers.
                 */
                if (!(cp->cache_flags & UMF_HASH)) {
                        chunksize = cp->cache_chunksize;
                        slabsize = cp->cache_slabsize;

                        umw->umw_ubase = mdb_alloc(slabsize +
                            sizeof (umem_bufctl_t), UM_SLEEP);

                        if (type & UM_ALLOCATED)
                                umw->umw_valid =
                                    mdb_alloc(slabsize / chunksize, UM_SLEEP);
                }
        }

        status = WALK_NEXT;

        if (mdb_layered_walk(layered, wsp) == -1) {
                mdb_warn("unable to start layered '%s' walk", layered);
                status = WALK_ERR;
        }

        if (status == WALK_ERR) {
                if (umw->umw_valid)
                        mdb_free(umw->umw_valid, slabsize / chunksize);

                if (umw->umw_ubase)
                        mdb_free(umw->umw_ubase, slabsize +
                            sizeof (umem_bufctl_t));

                if (umw->umw_maglist)
                        mdb_free(umw->umw_maglist, umw->umw_max *
                            sizeof (uintptr_t));

                mdb_free(umw, sizeof (umem_walk_t));
                wsp->walk_data = NULL;
        }

out2:
        if (status == WALK_ERR)
                mdb_free(cp, csize);

        return (status);
}

int
umem_walk_step(mdb_walk_state_t *wsp)
{
        umem_walk_t *umw = wsp->walk_data;
        int type = umw->umw_type;
        umem_cache_t *cp = umw->umw_cp;

        void **maglist = umw->umw_maglist;
        int magcnt = umw->umw_count;

        uintptr_t chunksize, slabsize;
        uintptr_t addr;
        const umem_slab_t *sp;
        const umem_bufctl_t *bcp;
        umem_bufctl_t bc;

        int chunks;
        char *kbase;
        void *buf;
        int i, ret;

        char *valid, *ubase;

        /*
         * first, handle the 'umem_hash' layered walk case
         */
        if (type & UM_HASH) {
                /*
                 * We have a buffer which has been allocated out of the
                 * global layer. We need to make sure that it's not
                 * actually sitting in a magazine before we report it as
                 * an allocated buffer.
                 */
                buf = ((const umem_bufctl_t *)wsp->walk_layer)->bc_addr;

                if (magcnt > 0 &&
                    bsearch(&buf, maglist, magcnt, sizeof (void *),
                    addrcmp) != NULL)
                        return (WALK_NEXT);

                if (type & UM_BUFCTL)
                        return (bufctl_walk_callback(cp, wsp, wsp->walk_addr));

                return (umem_walk_callback(wsp, (uintptr_t)buf));
        }

        ret = WALK_NEXT;

        addr = umw->umw_addr;

        /*
         * If we're walking freed buffers, report everything in the
         * magazine layer before processing the first slab.
         */
        if ((type & UM_FREE) && magcnt != 0) {
                umw->umw_count = 0;             /* only do this once */
                for (i = 0; i < magcnt; i++) {
                        buf = maglist[i];

                        if (type & UM_BUFCTL) {
                                uintptr_t out;

                                if (cp->cache_flags & UMF_BUFTAG) {
                                        umem_buftag_t *btp;
                                        umem_buftag_t tag;

                                        /* LINTED - alignment */
                                        btp = UMEM_BUFTAG(cp, buf);
                                        if (mdb_vread(&tag, sizeof (tag),
                                            (uintptr_t)btp) == -1) {
                                                mdb_warn("reading buftag for "
                                                    "%p at %p", buf, btp);
                                                continue;
                                        }
                                        out = (uintptr_t)tag.bt_bufctl;
                                } else {
                                        if (umem_hash_lookup(cp, addr, buf,
                                            &out) == -1)
                                                continue;
                                }
                                ret = bufctl_walk_callback(cp, wsp, out);
                        } else {
                                ret = umem_walk_callback(wsp, (uintptr_t)buf);
                        }

                        if (ret != WALK_NEXT)
                                return (ret);
                }
        }

        /*
         * Handle the buffers in the current slab
         */
        chunksize = cp->cache_chunksize;
        slabsize = cp->cache_slabsize;

        sp = wsp->walk_layer;
        chunks = sp->slab_chunks;
        kbase = sp->slab_base;

        dprintf(("kbase is %p\n", kbase));

        if (!(cp->cache_flags & UMF_HASH)) {
                valid = umw->umw_valid;
                ubase = umw->umw_ubase;

                if (mdb_vread(ubase, chunks * chunksize,
                    (uintptr_t)kbase) == -1) {
                        mdb_warn("failed to read slab contents at %p", kbase);
                        return (WALK_ERR);
                }

                /*
                 * Set up the valid map as fully allocated -- we'll punch
                 * out the freelist.
                 */
                if (type & UM_ALLOCATED)
                        (void) memset(valid, 1, chunks);
        } else {
                valid = NULL;
                ubase = NULL;
        }

        /*
         * walk the slab's freelist
         */
        bcp = sp->slab_head;

        dprintf(("refcnt is %d; chunks is %d\n", sp->slab_refcnt, chunks));

        /*
         * since we could be in the middle of allocating a buffer,
         * our refcnt could be one higher than it aught.  So we
         * check one further on the freelist than the count allows.
         */
        for (i = sp->slab_refcnt; i <= chunks; i++) {
                uint_t ndx;

                dprintf(("bcp is %p\n", bcp));

                if (bcp == NULL) {
                        if (i == chunks)
                                break;
                        mdb_warn(
                            "slab %p in cache %p freelist too short by %d\n",
                            sp, addr, chunks - i);
                        break;
                }

                if (cp->cache_flags & UMF_HASH) {
                        if (mdb_vread(&bc, sizeof (bc), (uintptr_t)bcp) == -1) {
                                mdb_warn("failed to read bufctl ptr at %p",
                                    bcp);
                                break;
                        }
                        buf = bc.bc_addr;
                } else {
                        /*
                         * Otherwise the buffer is (or should be) in the slab
                         * that we've read in; determine its offset in the
                         * slab, validate that it's not corrupt, and add to
                         * our base address to find the umem_bufctl_t.  (Note
                         * that we don't need to add the size of the bufctl
                         * to our offset calculation because of the slop that's
                         * allocated for the buffer at ubase.)
                         */
                        uintptr_t offs = (uintptr_t)bcp - (uintptr_t)kbase;

                        if (offs > chunks * chunksize) {
                                mdb_warn("found corrupt bufctl ptr %p"
                                    " in slab %p in cache %p\n", bcp,
                                    wsp->walk_addr, addr);
                                break;
                        }

                        bc = *((umem_bufctl_t *)((uintptr_t)ubase + offs));
                        buf = UMEM_BUF(cp, bcp);
                }

                ndx = ((uintptr_t)buf - (uintptr_t)kbase) / chunksize;

                if (ndx > slabsize / cp->cache_bufsize) {
                        /*
                         * This is very wrong; we have managed to find
                         * a buffer in the slab which shouldn't
                         * actually be here.  Emit a warning, and
                         * try to continue.
                         */
                        mdb_warn("buf %p is out of range for "
                            "slab %p, cache %p\n", buf, sp, addr);
                } else if (type & UM_ALLOCATED) {
                        /*
                         * we have found a buffer on the slab's freelist;
                         * clear its entry
                         */
                        valid[ndx] = 0;
                } else {
                        /*
                         * Report this freed buffer
                         */
                        if (type & UM_BUFCTL) {
                                ret = bufctl_walk_callback(cp, wsp,
                                    (uintptr_t)bcp);
                        } else {
                                ret = umem_walk_callback(wsp, (uintptr_t)buf);
                        }
                        if (ret != WALK_NEXT)
                                return (ret);
                }

                bcp = bc.bc_next;
        }

        if (bcp != NULL) {
                dprintf(("slab %p in cache %p freelist too long (%p)\n",
                    sp, addr, bcp));
        }

        /*
         * If we are walking freed buffers, the loop above handled reporting
         * them.
         */
        if (type & UM_FREE)
                return (WALK_NEXT);

        if (type & UM_BUFCTL) {
                mdb_warn("impossible situation: small-slab UM_BUFCTL walk for "
                    "cache %p\n", addr);
                return (WALK_ERR);
        }

        /*
         * Report allocated buffers, skipping buffers in the magazine layer.
         * We only get this far for small-slab caches.
         */
        for (i = 0; ret == WALK_NEXT && i < chunks; i++) {
                buf = (char *)kbase + i * chunksize;

                if (!valid[i])
                        continue;               /* on slab freelist */

                if (magcnt > 0 &&
                    bsearch(&buf, maglist, magcnt, sizeof (void *),
                    addrcmp) != NULL)
                        continue;               /* in magazine layer */

                ret = umem_walk_callback(wsp, (uintptr_t)buf);
        }
        return (ret);
}

void
umem_walk_fini(mdb_walk_state_t *wsp)
{
        umem_walk_t *umw = wsp->walk_data;
        uintptr_t chunksize;
        uintptr_t slabsize;

        if (umw == NULL)
                return;

        if (umw->umw_maglist != NULL)
                mdb_free(umw->umw_maglist, umw->umw_max * sizeof (void *));

        chunksize = umw->umw_cp->cache_chunksize;
        slabsize = umw->umw_cp->cache_slabsize;

        if (umw->umw_valid != NULL)
                mdb_free(umw->umw_valid, slabsize / chunksize);
        if (umw->umw_ubase != NULL)
                mdb_free(umw->umw_ubase, slabsize + sizeof (umem_bufctl_t));

        mdb_free(umw->umw_cp, umw->umw_csize);
        mdb_free(umw, sizeof (umem_walk_t));
}

/*ARGSUSED*/
static int
umem_walk_all(uintptr_t addr, const umem_cache_t *c, mdb_walk_state_t *wsp)
{
        /*
         * Buffers allocated from NOTOUCH caches can also show up as freed
         * memory in other caches.  This can be a little confusing, so we
         * don't walk NOTOUCH caches when walking all caches (thereby assuring
         * that "::walk umem" and "::walk freemem" yield disjoint output).
         */
        if (c->cache_cflags & UMC_NOTOUCH)
                return (WALK_NEXT);

        if (mdb_pwalk(wsp->walk_data, wsp->walk_callback,
            wsp->walk_cbdata, addr) == -1)
                return (WALK_DONE);

        return (WALK_NEXT);
}

#define UMEM_WALK_ALL(name, wsp) { \
        wsp->walk_data = (name); \
        if (mdb_walk("umem_cache", (mdb_walk_cb_t)umem_walk_all, wsp) == -1) \
                return (WALK_ERR); \
        return (WALK_DONE); \
}

int
umem_walk_init(mdb_walk_state_t *wsp)
{
        if (wsp->walk_arg != NULL)
                wsp->walk_addr = (uintptr_t)wsp->walk_arg;

        if (wsp->walk_addr == 0)
                UMEM_WALK_ALL("umem", wsp);
        return (umem_walk_init_common(wsp, UM_ALLOCATED));
}

int
bufctl_walk_init(mdb_walk_state_t *wsp)
{
        if (wsp->walk_addr == 0)
                UMEM_WALK_ALL("bufctl", wsp);
        return (umem_walk_init_common(wsp, UM_ALLOCATED | UM_BUFCTL));
}

int
freemem_walk_init(mdb_walk_state_t *wsp)
{
        if (wsp->walk_addr == 0)
                UMEM_WALK_ALL("freemem", wsp);
        return (umem_walk_init_common(wsp, UM_FREE));
}

int
freectl_walk_init(mdb_walk_state_t *wsp)
{
        if (wsp->walk_addr == 0)
                UMEM_WALK_ALL("freectl", wsp);
        return (umem_walk_init_common(wsp, UM_FREE | UM_BUFCTL));
}

typedef struct bufctl_history_walk {
        void            *bhw_next;
        umem_cache_t    *bhw_cache;
        umem_slab_t     *bhw_slab;
        hrtime_t        bhw_timestamp;
} bufctl_history_walk_t;

int
bufctl_history_walk_init(mdb_walk_state_t *wsp)
{
        bufctl_history_walk_t *bhw;
        umem_bufctl_audit_t bc;
        umem_bufctl_audit_t bcn;

        if (wsp->walk_addr == 0) {
                mdb_warn("bufctl_history walk doesn't support global walks\n");
                return (WALK_ERR);
        }

        if (mdb_vread(&bc, sizeof (bc), wsp->walk_addr) == -1) {
                mdb_warn("unable to read bufctl at %p", wsp->walk_addr);
                return (WALK_ERR);
        }

        bhw = mdb_zalloc(sizeof (*bhw), UM_SLEEP);
        bhw->bhw_timestamp = 0;
        bhw->bhw_cache = bc.bc_cache;
        bhw->bhw_slab = bc.bc_slab;

        /*
         * sometimes the first log entry matches the base bufctl;  in that
         * case, skip the base bufctl.
         */
        if (bc.bc_lastlog != NULL &&
            mdb_vread(&bcn, sizeof (bcn), (uintptr_t)bc.bc_lastlog) != -1 &&
            bc.bc_addr == bcn.bc_addr &&
            bc.bc_cache == bcn.bc_cache &&
            bc.bc_slab == bcn.bc_slab &&
            bc.bc_timestamp == bcn.bc_timestamp &&
            bc.bc_thread == bcn.bc_thread)
                bhw->bhw_next = bc.bc_lastlog;
        else
                bhw->bhw_next = (void *)wsp->walk_addr;

        wsp->walk_addr = (uintptr_t)bc.bc_addr;
        wsp->walk_data = bhw;

        return (WALK_NEXT);
}

int
bufctl_history_walk_step(mdb_walk_state_t *wsp)
{
        bufctl_history_walk_t *bhw = wsp->walk_data;
        uintptr_t addr = (uintptr_t)bhw->bhw_next;
        uintptr_t baseaddr = wsp->walk_addr;
        umem_bufctl_audit_t *b;
        UMEM_LOCAL_BUFCTL_AUDIT(&b);

        if (addr == 0)
                return (WALK_DONE);

        if (mdb_vread(b, UMEM_BUFCTL_AUDIT_SIZE, addr) == -1) {
                mdb_warn("unable to read bufctl at %p", bhw->bhw_next);
                return (WALK_ERR);
        }

        /*
         * The bufctl is only valid if the address, cache, and slab are
         * correct.  We also check that the timestamp is decreasing, to
         * prevent infinite loops.
         */
        if ((uintptr_t)b->bc_addr != baseaddr ||
            b->bc_cache != bhw->bhw_cache ||
            b->bc_slab != bhw->bhw_slab ||
            (bhw->bhw_timestamp != 0 && b->bc_timestamp >= bhw->bhw_timestamp))
                return (WALK_DONE);

        bhw->bhw_next = b->bc_lastlog;
        bhw->bhw_timestamp = b->bc_timestamp;

        return (wsp->walk_callback(addr, b, wsp->walk_cbdata));
}

void
bufctl_history_walk_fini(mdb_walk_state_t *wsp)
{
        bufctl_history_walk_t *bhw = wsp->walk_data;

        mdb_free(bhw, sizeof (*bhw));
}

typedef struct umem_log_walk {
        umem_bufctl_audit_t *ulw_base;
        umem_bufctl_audit_t **ulw_sorted;
        umem_log_header_t ulw_lh;
        size_t ulw_size;
        size_t ulw_maxndx;
        size_t ulw_ndx;
} umem_log_walk_t;

int
umem_log_walk_init(mdb_walk_state_t *wsp)
{
        uintptr_t lp = wsp->walk_addr;
        umem_log_walk_t *ulw;
        umem_log_header_t *lhp;
        int maxndx, i, j, k;

        /*
         * By default (global walk), walk the umem_transaction_log.  Otherwise
         * read the log whose umem_log_header_t is stored at walk_addr.
         */
        if (lp == 0 && umem_readvar(&lp, "umem_transaction_log") == -1) {
                mdb_warn("failed to read 'umem_transaction_log'");
                return (WALK_ERR);
        }

        if (lp == 0) {
                mdb_warn("log is disabled\n");
                return (WALK_ERR);
        }

        ulw = mdb_zalloc(sizeof (umem_log_walk_t), UM_SLEEP);
        lhp = &ulw->ulw_lh;

        if (mdb_vread(lhp, sizeof (umem_log_header_t), lp) == -1) {
                mdb_warn("failed to read log header at %p", lp);
                mdb_free(ulw, sizeof (umem_log_walk_t));
                return (WALK_ERR);
        }

        ulw->ulw_size = lhp->lh_chunksize * lhp->lh_nchunks;
        ulw->ulw_base = mdb_alloc(ulw->ulw_size, UM_SLEEP);
        maxndx = lhp->lh_chunksize / UMEM_BUFCTL_AUDIT_SIZE - 1;

        if (mdb_vread(ulw->ulw_base, ulw->ulw_size,
            (uintptr_t)lhp->lh_base) == -1) {
                mdb_warn("failed to read log at base %p", lhp->lh_base);
                mdb_free(ulw->ulw_base, ulw->ulw_size);
                mdb_free(ulw, sizeof (umem_log_walk_t));
                return (WALK_ERR);
        }

        ulw->ulw_sorted = mdb_alloc(maxndx * lhp->lh_nchunks *
            sizeof (umem_bufctl_audit_t *), UM_SLEEP);

        for (i = 0, k = 0; i < lhp->lh_nchunks; i++) {
                caddr_t chunk = (caddr_t)
                    ((uintptr_t)ulw->ulw_base + i * lhp->lh_chunksize);

                for (j = 0; j < maxndx; j++) {
                        /* LINTED align */
                        ulw->ulw_sorted[k++] = (umem_bufctl_audit_t *)chunk;
                        chunk += UMEM_BUFCTL_AUDIT_SIZE;
                }
        }

        qsort(ulw->ulw_sorted, k, sizeof (umem_bufctl_audit_t *),
            (int(*)(const void *, const void *))bufctlcmp);

        ulw->ulw_maxndx = k;
        wsp->walk_data = ulw;

        return (WALK_NEXT);
}

int
umem_log_walk_step(mdb_walk_state_t *wsp)
{
        umem_log_walk_t *ulw = wsp->walk_data;
        umem_bufctl_audit_t *bcp;

        if (ulw->ulw_ndx == ulw->ulw_maxndx)
                return (WALK_DONE);

        bcp = ulw->ulw_sorted[ulw->ulw_ndx++];

        return (wsp->walk_callback((uintptr_t)bcp - (uintptr_t)ulw->ulw_base +
            (uintptr_t)ulw->ulw_lh.lh_base, bcp, wsp->walk_cbdata));
}

void
umem_log_walk_fini(mdb_walk_state_t *wsp)
{
        umem_log_walk_t *ulw = wsp->walk_data;

        mdb_free(ulw->ulw_base, ulw->ulw_size);
        mdb_free(ulw->ulw_sorted, ulw->ulw_maxndx *
            sizeof (umem_bufctl_audit_t *));
        mdb_free(ulw, sizeof (umem_log_walk_t));
}

typedef struct allocdby_bufctl {
        uintptr_t abb_addr;
        hrtime_t abb_ts;
} allocdby_bufctl_t;

typedef struct allocdby_walk {
        const char *abw_walk;
        uintptr_t abw_thread;
        size_t abw_nbufs;
        size_t abw_size;
        allocdby_bufctl_t *abw_buf;
        size_t abw_ndx;
} allocdby_walk_t;

int
allocdby_walk_bufctl(uintptr_t addr, const umem_bufctl_audit_t *bcp,
    allocdby_walk_t *abw)
{
        if ((uintptr_t)bcp->bc_thread != abw->abw_thread)
                return (WALK_NEXT);

        if (abw->abw_nbufs == abw->abw_size) {
                allocdby_bufctl_t *buf;
                size_t oldsize = sizeof (allocdby_bufctl_t) * abw->abw_size;

                buf = mdb_zalloc(oldsize << 1, UM_SLEEP);

                bcopy(abw->abw_buf, buf, oldsize);
                mdb_free(abw->abw_buf, oldsize);

                abw->abw_size <<= 1;
                abw->abw_buf = buf;
        }

        abw->abw_buf[abw->abw_nbufs].abb_addr = addr;
        abw->abw_buf[abw->abw_nbufs].abb_ts = bcp->bc_timestamp;
        abw->abw_nbufs++;

        return (WALK_NEXT);
}

/*ARGSUSED*/
int
allocdby_walk_cache(uintptr_t addr, const umem_cache_t *c, allocdby_walk_t *abw)
{
        if (mdb_pwalk(abw->abw_walk, (mdb_walk_cb_t)allocdby_walk_bufctl,
            abw, addr) == -1) {
                mdb_warn("couldn't walk bufctl for cache %p", addr);
                return (WALK_DONE);
        }

        return (WALK_NEXT);
}

static int
allocdby_cmp(const allocdby_bufctl_t *lhs, const allocdby_bufctl_t *rhs)
{
        if (lhs->abb_ts < rhs->abb_ts)
                return (1);
        if (lhs->abb_ts > rhs->abb_ts)
                return (-1);
        return (0);
}

static int
allocdby_walk_init_common(mdb_walk_state_t *wsp, const char *walk)
{
        allocdby_walk_t *abw;

        if (wsp->walk_addr == 0) {
                mdb_warn("allocdby walk doesn't support global walks\n");
                return (WALK_ERR);
        }

        abw = mdb_zalloc(sizeof (allocdby_walk_t), UM_SLEEP);

        abw->abw_thread = wsp->walk_addr;
        abw->abw_walk = walk;
        abw->abw_size = 128;    /* something reasonable */
        abw->abw_buf =
            mdb_zalloc(abw->abw_size * sizeof (allocdby_bufctl_t), UM_SLEEP);

        wsp->walk_data = abw;

        if (mdb_walk("umem_cache",
            (mdb_walk_cb_t)allocdby_walk_cache, abw) == -1) {
                mdb_warn("couldn't walk umem_cache");
                allocdby_walk_fini(wsp);
                return (WALK_ERR);
        }

        qsort(abw->abw_buf, abw->abw_nbufs, sizeof (allocdby_bufctl_t),
            (int(*)(const void *, const void *))allocdby_cmp);

        return (WALK_NEXT);
}

int
allocdby_walk_init(mdb_walk_state_t *wsp)
{
        return (allocdby_walk_init_common(wsp, "bufctl"));
}

int
freedby_walk_init(mdb_walk_state_t *wsp)
{
        return (allocdby_walk_init_common(wsp, "freectl"));
}

int
allocdby_walk_step(mdb_walk_state_t *wsp)
{
        allocdby_walk_t *abw = wsp->walk_data;
        uintptr_t addr;
        umem_bufctl_audit_t *bcp;
        UMEM_LOCAL_BUFCTL_AUDIT(&bcp);

        if (abw->abw_ndx == abw->abw_nbufs)
                return (WALK_DONE);

        addr = abw->abw_buf[abw->abw_ndx++].abb_addr;

        if (mdb_vread(bcp, UMEM_BUFCTL_AUDIT_SIZE, addr) == -1) {
                mdb_warn("couldn't read bufctl at %p", addr);
                return (WALK_DONE);
        }

        return (wsp->walk_callback(addr, bcp, wsp->walk_cbdata));
}

void
allocdby_walk_fini(mdb_walk_state_t *wsp)
{
        allocdby_walk_t *abw = wsp->walk_data;

        mdb_free(abw->abw_buf, sizeof (allocdby_bufctl_t) * abw->abw_size);
        mdb_free(abw, sizeof (allocdby_walk_t));
}

/*ARGSUSED*/
int
allocdby_walk(uintptr_t addr, const umem_bufctl_audit_t *bcp, void *ignored)
{
        char c[MDB_SYM_NAMLEN];
        GElf_Sym sym;
        int i;

        mdb_printf("%0?p %12llx ", addr, bcp->bc_timestamp);
        for (i = 0; i < bcp->bc_depth; i++) {
                if (mdb_lookup_by_addr(bcp->bc_stack[i],
                    MDB_SYM_FUZZY, c, sizeof (c), &sym) == -1)
                        continue;
                if (is_umem_sym(c, "umem_"))
                        continue;
                mdb_printf("%s+0x%lx",
                    c, bcp->bc_stack[i] - (uintptr_t)sym.st_value);
                break;
        }
        mdb_printf("\n");

        return (WALK_NEXT);
}

static int
allocdby_common(uintptr_t addr, uint_t flags, const char *w)
{
        if (!(flags & DCMD_ADDRSPEC))
                return (DCMD_USAGE);

        mdb_printf("%-?s %12s %s\n", "BUFCTL", "TIMESTAMP", "CALLER");

        if (mdb_pwalk(w, (mdb_walk_cb_t)allocdby_walk, NULL, addr) == -1) {
                mdb_warn("can't walk '%s' for %p", w, addr);
                return (DCMD_ERR);
        }

        return (DCMD_OK);
}

/*ARGSUSED*/
int
allocdby(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
        return (allocdby_common(addr, flags, "allocdby"));
}

/*ARGSUSED*/
int
freedby(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
        return (allocdby_common(addr, flags, "freedby"));
}

typedef struct whatis_info {
        mdb_whatis_t *wi_w;
        const umem_cache_t *wi_cache;
        const vmem_t *wi_vmem;
        vmem_t *wi_msb_arena;
        size_t wi_slab_size;
        int wi_slab_found;
        uint_t wi_freemem;
} whatis_info_t;

/* call one of our dcmd functions with "-v" and the provided address */
static void
whatis_call_printer(mdb_dcmd_f *dcmd, uintptr_t addr)
{
        mdb_arg_t a;
        a.a_type = MDB_TYPE_STRING;
        a.a_un.a_str = "-v";

        mdb_printf(":\n");
        (void) (*dcmd)(addr, DCMD_ADDRSPEC, 1, &a);
}

static void
whatis_print_umem(whatis_info_t *wi, uintptr_t maddr, uintptr_t addr,
    uintptr_t baddr)
{
        mdb_whatis_t *w = wi->wi_w;
        const umem_cache_t *cp = wi->wi_cache;
        int quiet = (mdb_whatis_flags(w) & WHATIS_QUIET);

        int call_printer = (!quiet && (cp->cache_flags & UMF_AUDIT));

        mdb_whatis_report_object(w, maddr, addr, "");

        if (baddr != 0 && !call_printer)
                mdb_printf("bufctl %p ", baddr);

        mdb_printf("%s from %s",
            (wi->wi_freemem == FALSE) ? "allocated" : "freed", cp->cache_name);

        if (call_printer && baddr != 0) {
                whatis_call_printer(bufctl, baddr);
                return;
        }
        mdb_printf("\n");
}

/*ARGSUSED*/
static int
whatis_walk_umem(uintptr_t addr, void *ignored, whatis_info_t *wi)
{
        mdb_whatis_t *w = wi->wi_w;

        uintptr_t cur;
        size_t size = wi->wi_cache->cache_bufsize;

        while (mdb_whatis_match(w, addr, size, &cur))
                whatis_print_umem(wi, cur, addr, 0);

        return (WHATIS_WALKRET(w));
}

/*ARGSUSED*/
static int
whatis_walk_bufctl(uintptr_t baddr, const umem_bufctl_t *bcp, whatis_info_t *wi)
{
        mdb_whatis_t *w = wi->wi_w;

        uintptr_t cur;
        uintptr_t addr = (uintptr_t)bcp->bc_addr;
        size_t size = wi->wi_cache->cache_bufsize;

        while (mdb_whatis_match(w, addr, size, &cur))
                whatis_print_umem(wi, cur, addr, baddr);

        return (WHATIS_WALKRET(w));
}


static int
whatis_walk_seg(uintptr_t addr, const vmem_seg_t *vs, whatis_info_t *wi)
{
        mdb_whatis_t *w = wi->wi_w;

        size_t size = vs->vs_end - vs->vs_start;
        uintptr_t cur;

        /* We're not interested in anything but alloc and free segments */
        if (vs->vs_type != VMEM_ALLOC && vs->vs_type != VMEM_FREE)
                return (WALK_NEXT);

        while (mdb_whatis_match(w, vs->vs_start, size, &cur)) {
                mdb_whatis_report_object(w, cur, vs->vs_start, "");

                /*
                 * If we're not printing it seperately, provide the vmem_seg
                 * pointer if it has a stack trace.
                 */
                if ((mdb_whatis_flags(w) & WHATIS_QUIET) &&
                    ((mdb_whatis_flags(w) & WHATIS_BUFCTL) != 0 ||
                    (vs->vs_type == VMEM_ALLOC && vs->vs_depth != 0))) {
                        mdb_printf("vmem_seg %p ", addr);
                }

                mdb_printf("%s from %s vmem arena",
                    (vs->vs_type == VMEM_ALLOC) ? "allocated" : "freed",
                    wi->wi_vmem->vm_name);

                if (!(mdb_whatis_flags(w) & WHATIS_QUIET))
                        whatis_call_printer(vmem_seg, addr);
                else
                        mdb_printf("\n");
        }

        return (WHATIS_WALKRET(w));
}

static int
whatis_walk_vmem(uintptr_t addr, const vmem_t *vmem, whatis_info_t *wi)
{
        mdb_whatis_t *w = wi->wi_w;
        const char *nm = vmem->vm_name;
        wi->wi_vmem = vmem;

        if (mdb_whatis_flags(w) & WHATIS_VERBOSE)
                mdb_printf("Searching vmem arena %s...\n", nm);

        if (mdb_pwalk("vmem_seg",
            (mdb_walk_cb_t)whatis_walk_seg, wi, addr) == -1) {
                mdb_warn("can't walk vmem seg for %p", addr);
                return (WALK_NEXT);
        }

        return (WHATIS_WALKRET(w));
}

/*ARGSUSED*/
static int
whatis_walk_slab(uintptr_t saddr, const umem_slab_t *sp, whatis_info_t *wi)
{
        mdb_whatis_t *w = wi->wi_w;

        /* It must overlap with the slab data, or it's not interesting */
        if (mdb_whatis_overlaps(w,
            (uintptr_t)sp->slab_base, wi->wi_slab_size)) {
                wi->wi_slab_found++;
                return (WALK_DONE);
        }
        return (WALK_NEXT);
}

static int
whatis_walk_cache(uintptr_t addr, const umem_cache_t *c, whatis_info_t *wi)
{
        mdb_whatis_t *w = wi->wi_w;
        char *walk, *freewalk;
        mdb_walk_cb_t func;
        int do_bufctl;

        /* Override the '-b' flag as necessary */
        if (!(c->cache_flags & UMF_HASH))
                do_bufctl = FALSE;      /* no bufctls to walk */
        else if (c->cache_flags & UMF_AUDIT)
                do_bufctl = TRUE;       /* we always want debugging info */
        else
                do_bufctl = ((mdb_whatis_flags(w) & WHATIS_BUFCTL) != 0);

        if (do_bufctl) {
                walk = "bufctl";
                freewalk = "freectl";
                func = (mdb_walk_cb_t)whatis_walk_bufctl;
        } else {
                walk = "umem";
                freewalk = "freemem";
                func = (mdb_walk_cb_t)whatis_walk_umem;
        }

        wi->wi_cache = c;

        if (mdb_whatis_flags(w) & WHATIS_VERBOSE)
                mdb_printf("Searching %s...\n", c->cache_name);

        /*
         * If more then two buffers live on each slab, figure out if we're
         * interested in anything in any slab before doing the more expensive
         * umem/freemem (bufctl/freectl) walkers.
         */
        wi->wi_slab_size = c->cache_slabsize - c->cache_maxcolor;
        if (!(c->cache_flags & UMF_HASH))
                wi->wi_slab_size -= sizeof (umem_slab_t);

        if ((wi->wi_slab_size / c->cache_chunksize) > 2) {
                wi->wi_slab_found = 0;
                if (mdb_pwalk("umem_slab", (mdb_walk_cb_t)whatis_walk_slab, wi,
                    addr) == -1) {
                        mdb_warn("can't find umem_slab walker");
                        return (WALK_DONE);
                }
                if (wi->wi_slab_found == 0)
                        return (WALK_NEXT);
        }

        wi->wi_freemem = FALSE;
        if (mdb_pwalk(walk, func, wi, addr) == -1) {
                mdb_warn("can't find %s walker", walk);
                return (WALK_DONE);
        }

        if (mdb_whatis_done(w))
                return (WALK_DONE);

        /*
         * We have searched for allocated memory; now search for freed memory.
         */
        if (mdb_whatis_flags(w) & WHATIS_VERBOSE)
                mdb_printf("Searching %s for free memory...\n", c->cache_name);

        wi->wi_freemem = TRUE;

        if (mdb_pwalk(freewalk, func, wi, addr) == -1) {
                mdb_warn("can't find %s walker", freewalk);
                return (WALK_DONE);
        }

        return (WHATIS_WALKRET(w));
}

static int
whatis_walk_touch(uintptr_t addr, const umem_cache_t *c, whatis_info_t *wi)
{
        if (c->cache_arena == wi->wi_msb_arena ||
            (c->cache_cflags & UMC_NOTOUCH))
                return (WALK_NEXT);

        return (whatis_walk_cache(addr, c, wi));
}

static int
whatis_walk_metadata(uintptr_t addr, const umem_cache_t *c, whatis_info_t *wi)
{
        if (c->cache_arena != wi->wi_msb_arena)
                return (WALK_NEXT);

        return (whatis_walk_cache(addr, c, wi));
}

static int
whatis_walk_notouch(uintptr_t addr, const umem_cache_t *c, whatis_info_t *wi)
{
        if (c->cache_arena == wi->wi_msb_arena ||
            !(c->cache_cflags & UMC_NOTOUCH))
                return (WALK_NEXT);

        return (whatis_walk_cache(addr, c, wi));
}

/*ARGSUSED*/
static int
whatis_run_umem(mdb_whatis_t *w, void *ignored)
{
        whatis_info_t wi;

        bzero(&wi, sizeof (wi));
        wi.wi_w = w;

        /* umem's metadata is allocated from the umem_internal_arena */
        if (umem_readvar(&wi.wi_msb_arena, "umem_internal_arena") == -1)
                mdb_warn("unable to readvar \"umem_internal_arena\"");

        /*
         * We process umem caches in the following order:
         *
         *      non-UMC_NOTOUCH, non-metadata   (typically the most interesting)
         *      metadata                        (can be huge with UMF_AUDIT)
         *      UMC_NOTOUCH, non-metadata       (see umem_walk_all())
         */
        if (mdb_walk("umem_cache", (mdb_walk_cb_t)whatis_walk_touch,
            &wi) == -1 ||
            mdb_walk("umem_cache", (mdb_walk_cb_t)whatis_walk_metadata,
            &wi) == -1 ||
            mdb_walk("umem_cache", (mdb_walk_cb_t)whatis_walk_notouch,
            &wi) == -1) {
                mdb_warn("couldn't find umem_cache walker");
                return (1);
        }
        return (0);
}

/*ARGSUSED*/
static int
whatis_run_vmem(mdb_whatis_t *w, void *ignored)
{
        whatis_info_t wi;

        bzero(&wi, sizeof (wi));
        wi.wi_w = w;

        if (mdb_walk("vmem_postfix",
            (mdb_walk_cb_t)whatis_walk_vmem, &wi) == -1) {
                mdb_warn("couldn't find vmem_postfix walker");
                return (1);
        }
        return (0);
}

int
umem_init(void)
{
        mdb_walker_t w = {
                "umem_cache", "walk list of umem caches", umem_cache_walk_init,
                umem_cache_walk_step, umem_cache_walk_fini
        };

        if (mdb_add_walker(&w) == -1) {
                mdb_warn("failed to add umem_cache walker");
                return (-1);
        }

        if (umem_update_variables() == -1)
                return (-1);

        /* install a callback so that our variables are always up-to-date */
        (void) mdb_callback_add(MDB_CALLBACK_STCHG, umem_statechange_cb, NULL);
        umem_statechange_cb(NULL);

        /*
         * Register our ::whatis callbacks.
         */
        mdb_whatis_register("umem", whatis_run_umem, NULL,
            WHATIS_PRIO_ALLOCATOR, WHATIS_REG_NO_ID);
        mdb_whatis_register("vmem", whatis_run_vmem, NULL,
            WHATIS_PRIO_ALLOCATOR, WHATIS_REG_NO_ID);

        return (0);
}

typedef struct umem_log_cpu {
        uintptr_t umc_low;
        uintptr_t umc_high;
} umem_log_cpu_t;

int
umem_log_walk(uintptr_t addr, const umem_bufctl_audit_t *b, umem_log_cpu_t *umc)
{
        int i;

        for (i = 0; i < umem_max_ncpus; i++) {
                if (addr >= umc[i].umc_low && addr < umc[i].umc_high)
                        break;
        }

        if (i == umem_max_ncpus)
                mdb_printf("   ");
        else
                mdb_printf("%3d", i);

        mdb_printf(" %0?p %0?p %16llx %0?p\n", addr, b->bc_addr,
            b->bc_timestamp, b->bc_thread);

        return (WALK_NEXT);
}

/*ARGSUSED*/
int
umem_log(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
        umem_log_header_t lh;
        umem_cpu_log_header_t clh;
        uintptr_t lhp, clhp;
        umem_log_cpu_t *umc;
        int i;

        if (umem_readvar(&lhp, "umem_transaction_log") == -1) {
                mdb_warn("failed to read 'umem_transaction_log'");
                return (DCMD_ERR);
        }

        if (lhp == 0) {
                mdb_warn("no umem transaction log\n");
                return (DCMD_ERR);
        }

        if (mdb_vread(&lh, sizeof (umem_log_header_t), lhp) == -1) {
                mdb_warn("failed to read log header at %p", lhp);
                return (DCMD_ERR);
        }

        clhp = lhp + ((uintptr_t)&lh.lh_cpu[0] - (uintptr_t)&lh);

        umc = mdb_zalloc(sizeof (umem_log_cpu_t) * umem_max_ncpus,
            UM_SLEEP | UM_GC);

        for (i = 0; i < umem_max_ncpus; i++) {
                if (mdb_vread(&clh, sizeof (clh), clhp) == -1) {
                        mdb_warn("cannot read cpu %d's log header at %p",
                            i, clhp);
                        return (DCMD_ERR);
                }

                umc[i].umc_low = clh.clh_chunk * lh.lh_chunksize +
                    (uintptr_t)lh.lh_base;
                umc[i].umc_high = (uintptr_t)clh.clh_current;

                clhp += sizeof (umem_cpu_log_header_t);
        }

        if (DCMD_HDRSPEC(flags)) {
                mdb_printf("%3s %-?s %-?s %16s %-?s\n", "CPU", "ADDR",
                    "BUFADDR", "TIMESTAMP", "THREAD");
        }

        /*
         * If we have been passed an address, we'll just print out that
         * log entry.
         */
        if (flags & DCMD_ADDRSPEC) {
                umem_bufctl_audit_t *bp;
                UMEM_LOCAL_BUFCTL_AUDIT(&bp);

                if (mdb_vread(bp, UMEM_BUFCTL_AUDIT_SIZE, addr) == -1) {
                        mdb_warn("failed to read bufctl at %p", addr);
                        return (DCMD_ERR);
                }

                (void) umem_log_walk(addr, bp, umc);

                return (DCMD_OK);
        }

        if (mdb_walk("umem_log", (mdb_walk_cb_t)umem_log_walk, umc) == -1) {
                mdb_warn("can't find umem log walker");
                return (DCMD_ERR);
        }

        return (DCMD_OK);
}

typedef struct bufctl_history_cb {
        int             bhc_flags;
        int             bhc_argc;
        const mdb_arg_t *bhc_argv;
        int             bhc_ret;
} bufctl_history_cb_t;

/*ARGSUSED*/
static int
bufctl_history_callback(uintptr_t addr, const void *ign, void *arg)
{
        bufctl_history_cb_t *bhc = arg;

        bhc->bhc_ret =
            bufctl(addr, bhc->bhc_flags, bhc->bhc_argc, bhc->bhc_argv);

        bhc->bhc_flags &= ~DCMD_LOOPFIRST;

        return ((bhc->bhc_ret == DCMD_OK)? WALK_NEXT : WALK_DONE);
}

void
bufctl_help(void)
{
        mdb_printf("%s\n",
"Display the contents of umem_bufctl_audit_ts, with optional filtering.\n");
        mdb_dec_indent(2);
        mdb_printf("%<b>OPTIONS%</b>\n");
        mdb_inc_indent(2);
        mdb_printf("%s",
"  -v    Display the full content of the bufctl, including its stack trace\n"
"  -h    retrieve the bufctl's transaction history, if available\n"
"  -a addr\n"
"        filter out bufctls not involving the buffer at addr\n"
"  -c caller\n"
"        filter out bufctls without the function/PC in their stack trace\n"
"  -e earliest\n"
"        filter out bufctls timestamped before earliest\n"
"  -l latest\n"
"        filter out bufctls timestamped after latest\n"
"  -t thread\n"
"        filter out bufctls not involving thread\n");
}

int
bufctl(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
        uint_t verbose = FALSE;
        uint_t history = FALSE;
        uint_t in_history = FALSE;
        uintptr_t caller = 0, thread = 0;
        uintptr_t laddr, haddr, baddr = 0;
        hrtime_t earliest = 0, latest = 0;
        int i, depth;
        char c[MDB_SYM_NAMLEN];
        GElf_Sym sym;
        umem_bufctl_audit_t *bcp;
        UMEM_LOCAL_BUFCTL_AUDIT(&bcp);

        if (mdb_getopts(argc, argv,
            'v', MDB_OPT_SETBITS, TRUE, &verbose,
            'h', MDB_OPT_SETBITS, TRUE, &history,
            'H', MDB_OPT_SETBITS, TRUE, &in_history,            /* internal */
            'c', MDB_OPT_UINTPTR, &caller,
            't', MDB_OPT_UINTPTR, &thread,
            'e', MDB_OPT_UINT64, &earliest,
            'l', MDB_OPT_UINT64, &latest,
            'a', MDB_OPT_UINTPTR, &baddr, NULL) != argc)
                return (DCMD_USAGE);

        if (!(flags & DCMD_ADDRSPEC))
                return (DCMD_USAGE);

        if (in_history && !history)
                return (DCMD_USAGE);

        if (history && !in_history) {
                mdb_arg_t *nargv = mdb_zalloc(sizeof (*nargv) * (argc + 1),
                    UM_SLEEP | UM_GC);
                bufctl_history_cb_t bhc;

                nargv[0].a_type = MDB_TYPE_STRING;
                nargv[0].a_un.a_str = "-H";             /* prevent recursion */

                for (i = 0; i < argc; i++)
                        nargv[i + 1] = argv[i];

                /*
                 * When in history mode, we treat each element as if it
                 * were in a seperate loop, so that the headers group
                 * bufctls with similar histories.
                 */
                bhc.bhc_flags = flags | DCMD_LOOP | DCMD_LOOPFIRST;
                bhc.bhc_argc = argc + 1;
                bhc.bhc_argv = nargv;
                bhc.bhc_ret = DCMD_OK;

                if (mdb_pwalk("bufctl_history", bufctl_history_callback, &bhc,
                    addr) == -1) {
                        mdb_warn("unable to walk bufctl_history");
                        return (DCMD_ERR);
                }

                if (bhc.bhc_ret == DCMD_OK && !(flags & DCMD_PIPE_OUT))
                        mdb_printf("\n");

                return (bhc.bhc_ret);
        }

        if (DCMD_HDRSPEC(flags) && !(flags & DCMD_PIPE_OUT)) {
                if (verbose) {
                        mdb_printf("%16s %16s %16s %16s\n"
                            "%<u>%16s %16s %16s %16s%</u>\n",
                            "ADDR", "BUFADDR", "TIMESTAMP", "THREAD",
                            "", "CACHE", "LASTLOG", "CONTENTS");
                } else {
                        mdb_printf("%<u>%-?s %-?s %-12s %5s %s%</u>\n",
                            "ADDR", "BUFADDR", "TIMESTAMP", "THRD", "CALLER");
                }
        }

        if (mdb_vread(bcp, UMEM_BUFCTL_AUDIT_SIZE, addr) == -1) {
                mdb_warn("couldn't read bufctl at %p", addr);
                return (DCMD_ERR);
        }

        /*
         * Guard against bogus bc_depth in case the bufctl is corrupt or
         * the address does not really refer to a bufctl.
         */
        depth = MIN(bcp->bc_depth, umem_stack_depth);

        if (caller != 0) {
                laddr = caller;
                haddr = caller + sizeof (caller);

                if (mdb_lookup_by_addr(caller, MDB_SYM_FUZZY, c, sizeof (c),
                    &sym) != -1 && caller == (uintptr_t)sym.st_value) {
                        /*
                         * We were provided an exact symbol value; any
                         * address in the function is valid.
                         */
                        laddr = (uintptr_t)sym.st_value;
                        haddr = (uintptr_t)sym.st_value + sym.st_size;
                }

                for (i = 0; i < depth; i++)
                        if (bcp->bc_stack[i] >= laddr &&
                            bcp->bc_stack[i] < haddr)
                                break;

                if (i == depth)
                        return (DCMD_OK);
        }

        if (thread != 0 && (uintptr_t)bcp->bc_thread != thread)
                return (DCMD_OK);

        if (earliest != 0 && bcp->bc_timestamp < earliest)
                return (DCMD_OK);

        if (latest != 0 && bcp->bc_timestamp > latest)
                return (DCMD_OK);

        if (baddr != 0 && (uintptr_t)bcp->bc_addr != baddr)
                return (DCMD_OK);

        if (flags & DCMD_PIPE_OUT) {
                mdb_printf("%#r\n", addr);
                return (DCMD_OK);
        }

        if (verbose) {
                mdb_printf(
                    "%<b>%16p%</b> %16p %16llx %16d\n"
                    "%16s %16p %16p %16p\n",
                    addr, bcp->bc_addr, bcp->bc_timestamp, bcp->bc_thread,
                    "", bcp->bc_cache, bcp->bc_lastlog, bcp->bc_contents);

                mdb_inc_indent(17);
                for (i = 0; i < depth; i++)
                        mdb_printf("%a\n", bcp->bc_stack[i]);
                mdb_dec_indent(17);
                mdb_printf("\n");
        } else {
                mdb_printf("%0?p %0?p %12llx %5d", addr, bcp->bc_addr,
                    bcp->bc_timestamp, bcp->bc_thread);

                for (i = 0; i < depth; i++) {
                        if (mdb_lookup_by_addr(bcp->bc_stack[i],
                            MDB_SYM_FUZZY, c, sizeof (c), &sym) == -1)
                                continue;
                        if (is_umem_sym(c, "umem_"))
                                continue;
                        mdb_printf(" %a\n", bcp->bc_stack[i]);
                        break;
                }

                if (i >= depth)
                        mdb_printf("\n");
        }

        return (DCMD_OK);
}

/*ARGSUSED*/
int
bufctl_audit(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
        mdb_arg_t a;

        if (!(flags & DCMD_ADDRSPEC))
                return (DCMD_USAGE);

        if (argc != 0)
                return (DCMD_USAGE);

        a.a_type = MDB_TYPE_STRING;
        a.a_un.a_str = "-v";

        return (bufctl(addr, flags, 1, &a));
}

typedef struct umem_verify {
        uint64_t *umv_buf;              /* buffer to read cache contents into */
        size_t umv_size;                /* number of bytes in umv_buf */
        int umv_corruption;             /* > 0 if corruption found. */
        int umv_besilent;               /* report actual corruption sites */
        struct umem_cache umv_cache;    /* the cache we're operating on */
} umem_verify_t;

/*
 * verify_pattern()
 *      verify that buf is filled with the pattern pat.
 */
static int64_t
verify_pattern(uint64_t *buf_arg, size_t size, uint64_t pat)
{
        /*LINTED*/
        uint64_t *bufend = (uint64_t *)((char *)buf_arg + size);
        uint64_t *buf;

        for (buf = buf_arg; buf < bufend; buf++)
                if (*buf != pat)
                        return ((uintptr_t)buf - (uintptr_t)buf_arg);
        return (-1);
}

/*
 * verify_buftag()
 *      verify that btp->bt_bxstat == (bcp ^ pat)
 */
static int
verify_buftag(umem_buftag_t *btp, uintptr_t pat)
{
        return (btp->bt_bxstat == ((intptr_t)btp->bt_bufctl ^ pat) ? 0 : -1);
}

/*
 * verify_free()
 *      verify the integrity of a free block of memory by checking
 *      that it is filled with 0xdeadbeef and that its buftag is sane.
 */
/*ARGSUSED1*/
static int
verify_free(uintptr_t addr, const void *data, void *private)
{
        umem_verify_t *umv = (umem_verify_t *)private;
        uint64_t *buf = umv->umv_buf;   /* buf to validate */
        int64_t corrupt;                /* corruption offset */
        umem_buftag_t *buftagp;         /* ptr to buftag */
        umem_cache_t *cp = &umv->umv_cache;
        int besilent = umv->umv_besilent;

        /*LINTED*/
        buftagp = UMEM_BUFTAG(cp, buf);

        /*
         * Read the buffer to check.
         */
        if (mdb_vread(buf, umv->umv_size, addr) == -1) {
                if (!besilent)
                        mdb_warn("couldn't read %p", addr);
                return (WALK_NEXT);
        }

        if ((corrupt = verify_pattern(buf, cp->cache_verify,
            UMEM_FREE_PATTERN)) >= 0) {
                if (!besilent)
                        mdb_printf("buffer %p (free) seems corrupted, at %p\n",
                            addr, (uintptr_t)addr + corrupt);
                goto corrupt;
        }

        if ((cp->cache_flags & UMF_HASH) &&
            buftagp->bt_redzone != UMEM_REDZONE_PATTERN) {
                if (!besilent)
                        mdb_printf("buffer %p (free) seems to "
                            "have a corrupt redzone pattern\n", addr);
                goto corrupt;
        }

        /*
         * confirm bufctl pointer integrity.
         */
        if (verify_buftag(buftagp, UMEM_BUFTAG_FREE) == -1) {
                if (!besilent)
                        mdb_printf("buffer %p (free) has a corrupt "
                            "buftag\n", addr);
                goto corrupt;
        }

        return (WALK_NEXT);
corrupt:
        umv->umv_corruption++;
        return (WALK_NEXT);
}

/*
 * verify_alloc()
 *      Verify that the buftag of an allocated buffer makes sense with respect
 *      to the buffer.
 */
/*ARGSUSED1*/
static int
verify_alloc(uintptr_t addr, const void *data, void *private)
{
        umem_verify_t *umv = (umem_verify_t *)private;
        umem_cache_t *cp = &umv->umv_cache;
        uint64_t *buf = umv->umv_buf;   /* buf to validate */
        /*LINTED*/
        umem_buftag_t *buftagp = UMEM_BUFTAG(cp, buf);
        uint32_t *ip = (uint32_t *)buftagp;
        uint8_t *bp = (uint8_t *)buf;
        int looks_ok = 0, size_ok = 1;  /* flags for finding corruption */
        int besilent = umv->umv_besilent;

        /*
         * Read the buffer to check.
         */
        if (mdb_vread(buf, umv->umv_size, addr) == -1) {
                if (!besilent)
                        mdb_warn("couldn't read %p", addr);
                return (WALK_NEXT);
        }

        /*
         * There are two cases to handle:
         * 1. If the buf was alloc'd using umem_cache_alloc, it will have
         *    0xfeedfacefeedface at the end of it
         * 2. If the buf was alloc'd using umem_alloc, it will have
         *    0xbb just past the end of the region in use.  At the buftag,
         *    it will have 0xfeedface (or, if the whole buffer is in use,
         *    0xfeedface & bb000000 or 0xfeedfacf & 000000bb depending on
         *    endianness), followed by 32 bits containing the offset of the
         *    0xbb byte in the buffer.
         *
         * Finally, the two 32-bit words that comprise the second half of the
         * buftag should xor to UMEM_BUFTAG_ALLOC
         */

        if (buftagp->bt_redzone == UMEM_REDZONE_PATTERN)
                looks_ok = 1;
        else if (!UMEM_SIZE_VALID(ip[1]))
                size_ok = 0;
        else if (bp[UMEM_SIZE_DECODE(ip[1])] == UMEM_REDZONE_BYTE)
                looks_ok = 1;
        else
                size_ok = 0;

        if (!size_ok) {
                if (!besilent)
                        mdb_printf("buffer %p (allocated) has a corrupt "
                            "redzone size encoding\n", addr);
                goto corrupt;
        }

        if (!looks_ok) {
                if (!besilent)
                        mdb_printf("buffer %p (allocated) has a corrupt "
                            "redzone signature\n", addr);
                goto corrupt;
        }

        if (verify_buftag(buftagp, UMEM_BUFTAG_ALLOC) == -1) {
                if (!besilent)
                        mdb_printf("buffer %p (allocated) has a "
                            "corrupt buftag\n", addr);
                goto corrupt;
        }

        return (WALK_NEXT);
corrupt:
        umv->umv_corruption++;
        return (WALK_NEXT);
}

/*ARGSUSED2*/
int
umem_verify(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
        if (flags & DCMD_ADDRSPEC) {
                int check_alloc = 0, check_free = 0;
                umem_verify_t umv;

                if (mdb_vread(&umv.umv_cache, sizeof (umv.umv_cache),
                    addr) == -1) {
                        mdb_warn("couldn't read umem_cache %p", addr);
                        return (DCMD_ERR);
                }

                umv.umv_size = umv.umv_cache.cache_buftag +
                    sizeof (umem_buftag_t);
                umv.umv_buf = mdb_alloc(umv.umv_size, UM_SLEEP | UM_GC);
                umv.umv_corruption = 0;

                if ((umv.umv_cache.cache_flags & UMF_REDZONE)) {
                        check_alloc = 1;
                        if (umv.umv_cache.cache_flags & UMF_DEADBEEF)
                                check_free = 1;
                } else {
                        if (!(flags & DCMD_LOOP)) {
                                mdb_warn("cache %p (%s) does not have "
                                    "redzone checking enabled\n", addr,
                                    umv.umv_cache.cache_name);
                        }
                        return (DCMD_ERR);
                }

                if (flags & DCMD_LOOP) {
                        /*
                         * table mode, don't print out every corrupt buffer
                         */
                        umv.umv_besilent = 1;
                } else {
                        mdb_printf("Summary for cache '%s'\n",
                            umv.umv_cache.cache_name);
                        mdb_inc_indent(2);
                        umv.umv_besilent = 0;
                }

                if (check_alloc)
                        (void) mdb_pwalk("umem", verify_alloc, &umv, addr);
                if (check_free)
                        (void) mdb_pwalk("freemem", verify_free, &umv, addr);

                if (flags & DCMD_LOOP) {
                        if (umv.umv_corruption == 0) {
                                mdb_printf("%-*s %?p clean\n",
                                    UMEM_CACHE_NAMELEN,
                                    umv.umv_cache.cache_name, addr);
                        } else {
                                char *s = "";   /* optional s in "buffer[s]" */
                                if (umv.umv_corruption > 1)
                                        s = "s";

                                mdb_printf("%-*s %?p %d corrupt buffer%s\n",
                                    UMEM_CACHE_NAMELEN,
                                    umv.umv_cache.cache_name, addr,
                                    umv.umv_corruption, s);
                        }
                } else {
                        /*
                         * This is the more verbose mode, when the user has
                         * type addr::umem_verify.  If the cache was clean,
                         * nothing will have yet been printed. So say something.
                         */
                        if (umv.umv_corruption == 0)
                                mdb_printf("clean\n");

                        mdb_dec_indent(2);
                }
        } else {
                /*
                 * If the user didn't specify a cache to verify, we'll walk all
                 * umem_cache's, specifying ourself as a callback for each...
                 * this is the equivalent of '::walk umem_cache .::umem_verify'
                 */
                mdb_printf("%<u>%-*s %-?s %-20s%</b>\n", UMEM_CACHE_NAMELEN,
                    "Cache Name", "Addr", "Cache Integrity");
                (void) (mdb_walk_dcmd("umem_cache", "umem_verify", 0, NULL));
        }

        return (DCMD_OK);
}

typedef struct vmem_node {
        struct vmem_node *vn_next;
        struct vmem_node *vn_parent;
        struct vmem_node *vn_sibling;
        struct vmem_node *vn_children;
        uintptr_t vn_addr;
        int vn_marked;
        vmem_t vn_vmem;
} vmem_node_t;

typedef struct vmem_walk {
        vmem_node_t *vw_root;
        vmem_node_t *vw_current;
} vmem_walk_t;

int
vmem_walk_init(mdb_walk_state_t *wsp)
{
        uintptr_t vaddr, paddr;
        vmem_node_t *head = NULL, *root = NULL, *current = NULL, *parent, *vp;
        vmem_walk_t *vw;

        if (umem_readvar(&vaddr, "vmem_list") == -1) {
                mdb_warn("couldn't read 'vmem_list'");
                return (WALK_ERR);
        }

        while (vaddr != 0) {
                vp = mdb_zalloc(sizeof (vmem_node_t), UM_SLEEP);
                vp->vn_addr = vaddr;
                vp->vn_next = head;
                head = vp;

                if (vaddr == wsp->walk_addr)
                        current = vp;

                if (mdb_vread(&vp->vn_vmem, sizeof (vmem_t), vaddr) == -1) {
                        mdb_warn("couldn't read vmem_t at %p", vaddr);
                        goto err;
                }

                vaddr = (uintptr_t)vp->vn_vmem.vm_next;
        }

        for (vp = head; vp != NULL; vp = vp->vn_next) {

                if ((paddr = (uintptr_t)vp->vn_vmem.vm_source) == 0) {
                        vp->vn_sibling = root;
                        root = vp;
                        continue;
                }

                for (parent = head; parent != NULL; parent = parent->vn_next) {
                        if (parent->vn_addr != paddr)
                                continue;
                        vp->vn_sibling = parent->vn_children;
                        parent->vn_children = vp;
                        vp->vn_parent = parent;
                        break;
                }

                if (parent == NULL) {
                        mdb_warn("couldn't find %p's parent (%p)\n",
                            vp->vn_addr, paddr);
                        goto err;
                }
        }

        vw = mdb_zalloc(sizeof (vmem_walk_t), UM_SLEEP);
        vw->vw_root = root;

        if (current != NULL)
                vw->vw_current = current;
        else
                vw->vw_current = root;

        wsp->walk_data = vw;
        return (WALK_NEXT);
err:
        for (vp = head; head != NULL; vp = head) {
                head = vp->vn_next;
                mdb_free(vp, sizeof (vmem_node_t));
        }

        return (WALK_ERR);
}

int
vmem_walk_step(mdb_walk_state_t *wsp)
{
        vmem_walk_t *vw = wsp->walk_data;
        vmem_node_t *vp;
        int rval;

        if ((vp = vw->vw_current) == NULL)
                return (WALK_DONE);

        rval = wsp->walk_callback(vp->vn_addr, &vp->vn_vmem, wsp->walk_cbdata);

        if (vp->vn_children != NULL) {
                vw->vw_current = vp->vn_children;
                return (rval);
        }

        do {
                vw->vw_current = vp->vn_sibling;
                vp = vp->vn_parent;
        } while (vw->vw_current == NULL && vp != NULL);

        return (rval);
}

/*
 * The "vmem_postfix" walk walks the vmem arenas in post-fix order; all
 * children are visited before their parent.  We perform the postfix walk
 * iteratively (rather than recursively) to allow mdb to regain control
 * after each callback.
 */
int
vmem_postfix_walk_step(mdb_walk_state_t *wsp)
{
        vmem_walk_t *vw = wsp->walk_data;
        vmem_node_t *vp = vw->vw_current;
        int rval;

        /*
         * If this node is marked, then we know that we have already visited
         * all of its children.  If the node has any siblings, they need to
         * be visited next; otherwise, we need to visit the parent.  Note
         * that vp->vn_marked will only be zero on the first invocation of
         * the step function.
         */
        if (vp->vn_marked) {
                if (vp->vn_sibling != NULL)
                        vp = vp->vn_sibling;
                else if (vp->vn_parent != NULL)
                        vp = vp->vn_parent;
                else {
                        /*
                         * We have neither a parent, nor a sibling, and we
                         * have already been visited; we're done.
                         */
                        return (WALK_DONE);
                }
        }

        /*
         * Before we visit this node, visit its children.
         */
        while (vp->vn_children != NULL && !vp->vn_children->vn_marked)
                vp = vp->vn_children;

        vp->vn_marked = 1;
        vw->vw_current = vp;
        rval = wsp->walk_callback(vp->vn_addr, &vp->vn_vmem, wsp->walk_cbdata);

        return (rval);
}

void
vmem_walk_fini(mdb_walk_state_t *wsp)
{
        vmem_walk_t *vw = wsp->walk_data;
        vmem_node_t *root = vw->vw_root;
        int done;

        if (root == NULL)
                return;

        if ((vw->vw_root = root->vn_children) != NULL)
                vmem_walk_fini(wsp);

        vw->vw_root = root->vn_sibling;
        done = (root->vn_sibling == NULL && root->vn_parent == NULL);
        mdb_free(root, sizeof (vmem_node_t));

        if (done) {
                mdb_free(vw, sizeof (vmem_walk_t));
        } else {
                vmem_walk_fini(wsp);
        }
}

typedef struct vmem_seg_walk {
        uint8_t vsw_type;
        uintptr_t vsw_start;
        uintptr_t vsw_current;
} vmem_seg_walk_t;

/*ARGSUSED*/
int
vmem_seg_walk_common_init(mdb_walk_state_t *wsp, uint8_t type, char *name)
{
        vmem_seg_walk_t *vsw;

        if (wsp->walk_addr == 0) {
                mdb_warn("vmem_%s does not support global walks\n", name);
                return (WALK_ERR);
        }

        wsp->walk_data = vsw = mdb_alloc(sizeof (vmem_seg_walk_t), UM_SLEEP);

        vsw->vsw_type = type;
        vsw->vsw_start = wsp->walk_addr + OFFSETOF(vmem_t, vm_seg0);
        vsw->vsw_current = vsw->vsw_start;

        return (WALK_NEXT);
}

/*
 * vmem segments can't have type 0 (this should be added to vmem_impl.h).
 */
#define VMEM_NONE       0

int
vmem_alloc_walk_init(mdb_walk_state_t *wsp)
{
        return (vmem_seg_walk_common_init(wsp, VMEM_ALLOC, "alloc"));
}

int
vmem_free_walk_init(mdb_walk_state_t *wsp)
{
        return (vmem_seg_walk_common_init(wsp, VMEM_FREE, "free"));
}

int
vmem_span_walk_init(mdb_walk_state_t *wsp)
{
        return (vmem_seg_walk_common_init(wsp, VMEM_SPAN, "span"));
}

int
vmem_seg_walk_init(mdb_walk_state_t *wsp)
{
        return (vmem_seg_walk_common_init(wsp, VMEM_NONE, "seg"));
}

int
vmem_seg_walk_step(mdb_walk_state_t *wsp)
{
        vmem_seg_t seg;
        vmem_seg_walk_t *vsw = wsp->walk_data;
        uintptr_t addr = vsw->vsw_current;
        static size_t seg_size = 0;
        int rval;

        if (!seg_size) {
                if (umem_readvar(&seg_size, "vmem_seg_size") == -1) {
                        mdb_warn("failed to read 'vmem_seg_size'");
                        seg_size = sizeof (vmem_seg_t);
                }
        }

        if (seg_size < sizeof (seg))
                bzero((caddr_t)&seg + seg_size, sizeof (seg) - seg_size);

        if (mdb_vread(&seg, seg_size, addr) == -1) {
                mdb_warn("couldn't read vmem_seg at %p", addr);
                return (WALK_ERR);
        }

        vsw->vsw_current = (uintptr_t)seg.vs_anext;
        if (vsw->vsw_type != VMEM_NONE && seg.vs_type != vsw->vsw_type) {
                rval = WALK_NEXT;
        } else {
                rval = wsp->walk_callback(addr, &seg, wsp->walk_cbdata);
        }

        if (vsw->vsw_current == vsw->vsw_start)
                return (WALK_DONE);

        return (rval);
}

void
vmem_seg_walk_fini(mdb_walk_state_t *wsp)
{
        vmem_seg_walk_t *vsw = wsp->walk_data;

        mdb_free(vsw, sizeof (vmem_seg_walk_t));
}

#define VMEM_NAMEWIDTH  22

int
vmem(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
        vmem_t v, parent;
        uintptr_t paddr;
        int ident = 0;
        char c[VMEM_NAMEWIDTH];

        if (!(flags & DCMD_ADDRSPEC)) {
                if (mdb_walk_dcmd("vmem", "vmem", argc, argv) == -1) {
                        mdb_warn("can't walk vmem");
                        return (DCMD_ERR);
                }
                return (DCMD_OK);
        }

        if (DCMD_HDRSPEC(flags))
                mdb_printf("%-?s %-*s %10s %12s %9s %5s\n",
                    "ADDR", VMEM_NAMEWIDTH, "NAME", "INUSE",
                    "TOTAL", "SUCCEED", "FAIL");

        if (mdb_vread(&v, sizeof (v), addr) == -1) {
                mdb_warn("couldn't read vmem at %p", addr);
                return (DCMD_ERR);
        }

        for (paddr = (uintptr_t)v.vm_source; paddr != 0; ident += 2) {
                if (mdb_vread(&parent, sizeof (parent), paddr) == -1) {
                        mdb_warn("couldn't trace %p's ancestry", addr);
                        ident = 0;
                        break;
                }
                paddr = (uintptr_t)parent.vm_source;
        }

        (void) mdb_snprintf(c, VMEM_NAMEWIDTH, "%*s%s", ident, "", v.vm_name);

        mdb_printf("%0?p %-*s %10llu %12llu %9llu %5llu\n",
            addr, VMEM_NAMEWIDTH, c,
            v.vm_kstat.vk_mem_inuse, v.vm_kstat.vk_mem_total,
            v.vm_kstat.vk_alloc, v.vm_kstat.vk_fail);

        return (DCMD_OK);
}

void
vmem_seg_help(void)
{
        mdb_printf("%s\n",
"Display the contents of vmem_seg_ts, with optional filtering.\n"
"\n"
"A vmem_seg_t represents a range of addresses (or arbitrary numbers),\n"
"representing a single chunk of data.  Only ALLOC segments have debugging\n"
"information.\n");
        mdb_dec_indent(2);
        mdb_printf("%<b>OPTIONS%</b>\n");
        mdb_inc_indent(2);
        mdb_printf("%s",
"  -v    Display the full content of the vmem_seg, including its stack trace\n"
"  -s    report the size of the segment, instead of the end address\n"
"  -c caller\n"
"        filter out segments without the function/PC in their stack trace\n"
"  -e earliest\n"
"        filter out segments timestamped before earliest\n"
"  -l latest\n"
"        filter out segments timestamped after latest\n"
"  -m minsize\n"
"        filer out segments smaller than minsize\n"
"  -M maxsize\n"
"        filer out segments larger than maxsize\n"
"  -t thread\n"
"        filter out segments not involving thread\n"
"  -T type\n"
"        filter out segments not of type 'type'\n"
"        type is one of: ALLOC/FREE/SPAN/ROTOR/WALKER\n");
}


/*ARGSUSED*/
int
vmem_seg(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
        vmem_seg_t vs;
        uintptr_t *stk = vs.vs_stack;
        uintptr_t sz;
        uint8_t t;
        const char *type = NULL;
        GElf_Sym sym;
        char c[MDB_SYM_NAMLEN];
        int no_debug;
        int i;
        int depth;
        uintptr_t laddr, haddr;

        uintptr_t caller = 0, thread = 0;
        uintptr_t minsize = 0, maxsize = 0;

        hrtime_t earliest = 0, latest = 0;

        uint_t size = 0;
        uint_t verbose = 0;

        if (!(flags & DCMD_ADDRSPEC))
                return (DCMD_USAGE);

        if (mdb_getopts(argc, argv,
            'c', MDB_OPT_UINTPTR, &caller,
            'e', MDB_OPT_UINT64, &earliest,
            'l', MDB_OPT_UINT64, &latest,
            's', MDB_OPT_SETBITS, TRUE, &size,
            'm', MDB_OPT_UINTPTR, &minsize,
            'M', MDB_OPT_UINTPTR, &maxsize,
            't', MDB_OPT_UINTPTR, &thread,
            'T', MDB_OPT_STR, &type,
            'v', MDB_OPT_SETBITS, TRUE, &verbose,
            NULL) != argc)
                return (DCMD_USAGE);

        if (DCMD_HDRSPEC(flags) && !(flags & DCMD_PIPE_OUT)) {
                if (verbose) {
                        mdb_printf("%16s %4s %16s %16s %16s\n"
                            "%<u>%16s %4s %16s %16s %16s%</u>\n",
                            "ADDR", "TYPE", "START", "END", "SIZE",
                            "", "", "THREAD", "TIMESTAMP", "");
                } else {
                        mdb_printf("%?s %4s %?s %?s %s\n", "ADDR", "TYPE",
                            "START", size? "SIZE" : "END", "WHO");
                }
        }

        if (mdb_vread(&vs, sizeof (vs), addr) == -1) {
                mdb_warn("couldn't read vmem_seg at %p", addr);
                return (DCMD_ERR);
        }

        if (type != NULL) {
                if (strcmp(type, "ALLC") == 0 || strcmp(type, "ALLOC") == 0)
                        t = VMEM_ALLOC;
                else if (strcmp(type, "FREE") == 0)
                        t = VMEM_FREE;
                else if (strcmp(type, "SPAN") == 0)
                        t = VMEM_SPAN;
                else if (strcmp(type, "ROTR") == 0 ||
                    strcmp(type, "ROTOR") == 0)
                        t = VMEM_ROTOR;
                else if (strcmp(type, "WLKR") == 0 ||
                    strcmp(type, "WALKER") == 0)
                        t = VMEM_WALKER;
                else {
                        mdb_warn("\"%s\" is not a recognized vmem_seg type\n",
                            type);
                        return (DCMD_ERR);
                }

                if (vs.vs_type != t)
                        return (DCMD_OK);
        }

        sz = vs.vs_end - vs.vs_start;

        if (minsize != 0 && sz < minsize)
                return (DCMD_OK);

        if (maxsize != 0 && sz > maxsize)
                return (DCMD_OK);

        t = vs.vs_type;
        depth = vs.vs_depth;

        /*
         * debug info, when present, is only accurate for VMEM_ALLOC segments
         */
        no_debug = (t != VMEM_ALLOC) ||
            (depth == 0 || depth > VMEM_STACK_DEPTH);

        if (no_debug) {
                if (caller != 0 || thread != 0 || earliest != 0 || latest != 0)
                        return (DCMD_OK);               /* not enough info */
        } else {
                if (caller != 0) {
                        laddr = caller;
                        haddr = caller + sizeof (caller);

                        if (mdb_lookup_by_addr(caller, MDB_SYM_FUZZY, c,
                            sizeof (c), &sym) != -1 &&
                            caller == (uintptr_t)sym.st_value) {
                                /*
                                 * We were provided an exact symbol value; any
                                 * address in the function is valid.
                                 */
                                laddr = (uintptr_t)sym.st_value;
                                haddr = (uintptr_t)sym.st_value + sym.st_size;
                        }

                        for (i = 0; i < depth; i++)
                                if (vs.vs_stack[i] >= laddr &&
                                    vs.vs_stack[i] < haddr)
                                        break;

                        if (i == depth)
                                return (DCMD_OK);
                }

                if (thread != 0 && (uintptr_t)vs.vs_thread != thread)
                        return (DCMD_OK);

                if (earliest != 0 && vs.vs_timestamp < earliest)
                        return (DCMD_OK);

                if (latest != 0 && vs.vs_timestamp > latest)
                        return (DCMD_OK);
        }

        type = (t == VMEM_ALLOC ? "ALLC" :
            t == VMEM_FREE ? "FREE" :
            t == VMEM_SPAN ? "SPAN" :
            t == VMEM_ROTOR ? "ROTR" :
            t == VMEM_WALKER ? "WLKR" :
            "????");

        if (flags & DCMD_PIPE_OUT) {
                mdb_printf("%#r\n", addr);
                return (DCMD_OK);
        }

        if (verbose) {
                mdb_printf("%<b>%16p%</b> %4s %16p %16p %16d\n",
                    addr, type, vs.vs_start, vs.vs_end, sz);

                if (no_debug)
                        return (DCMD_OK);

                mdb_printf("%16s %4s %16d %16llx\n",
                    "", "", vs.vs_thread, vs.vs_timestamp);

                mdb_inc_indent(17);
                for (i = 0; i < depth; i++) {
                        mdb_printf("%a\n", stk[i]);
                }
                mdb_dec_indent(17);
                mdb_printf("\n");
        } else {
                mdb_printf("%0?p %4s %0?p %0?p", addr, type,
                    vs.vs_start, size? sz : vs.vs_end);

                if (no_debug) {
                        mdb_printf("\n");
                        return (DCMD_OK);
                }

                for (i = 0; i < depth; i++) {
                        if (mdb_lookup_by_addr(stk[i], MDB_SYM_FUZZY,
                            c, sizeof (c), &sym) == -1)
                                continue;
                        if (is_umem_sym(c, "vmem_"))
                                continue;
                        break;
                }
                mdb_printf(" %a\n", stk[i]);
        }
        return (DCMD_OK);
}

/*ARGSUSED*/
static int
showbc(uintptr_t addr, const umem_bufctl_audit_t *bcp, hrtime_t *newest)
{
        char name[UMEM_CACHE_NAMELEN + 1];
        hrtime_t delta;
        int i, depth;

        if (bcp->bc_timestamp == 0)
                return (WALK_DONE);

        if (*newest == 0)
                *newest = bcp->bc_timestamp;

        delta = *newest - bcp->bc_timestamp;
        depth = MIN(bcp->bc_depth, umem_stack_depth);

        if (mdb_readstr(name, sizeof (name), (uintptr_t)
            &bcp->bc_cache->cache_name) <= 0)
                (void) mdb_snprintf(name, sizeof (name), "%a", bcp->bc_cache);

        mdb_printf("\nT-%lld.%09lld  addr=%p  %s\n",
            delta / NANOSEC, delta % NANOSEC, bcp->bc_addr, name);

        for (i = 0; i < depth; i++)
                mdb_printf("\t %a\n", bcp->bc_stack[i]);

        return (WALK_NEXT);
}

int
umalog(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
        const char *logname = "umem_transaction_log";
        hrtime_t newest = 0;

        if ((flags & DCMD_ADDRSPEC) || argc > 1)
                return (DCMD_USAGE);

        if (argc > 0) {
                if (argv->a_type != MDB_TYPE_STRING)
                        return (DCMD_USAGE);
                if (strcmp(argv->a_un.a_str, "fail") == 0)
                        logname = "umem_failure_log";
                else if (strcmp(argv->a_un.a_str, "slab") == 0)
                        logname = "umem_slab_log";
                else
                        return (DCMD_USAGE);
        }

        if (umem_readvar(&addr, logname) == -1) {
                mdb_warn("failed to read %s log header pointer", logname);
                return (DCMD_ERR);
        }

        if (mdb_pwalk("umem_log", (mdb_walk_cb_t)showbc, &newest, addr) == -1) {
                mdb_warn("failed to walk umem log");
                return (DCMD_ERR);
        }

        return (DCMD_OK);
}

/*
 * As the final lure for die-hard crash(8) users, we provide ::umausers here.
 * The first piece is a structure which we use to accumulate umem_cache_t
 * addresses of interest.  The umc_add is used as a callback for the umem_cache
 * walker; we either add all caches, or ones named explicitly as arguments.
 */

typedef struct umclist {
        const char *umc_name;                   /* Name to match (or NULL) */
        uintptr_t *umc_caches;                  /* List of umem_cache_t addrs */
        int umc_nelems;                         /* Num entries in umc_caches */
        int umc_size;                           /* Size of umc_caches array */
} umclist_t;

static int
umc_add(uintptr_t addr, const umem_cache_t *cp, umclist_t *umc)
{
        void *p;
        int s;

        if (umc->umc_name == NULL ||
            strcmp(cp->cache_name, umc->umc_name) == 0) {
                /*
                 * If we have a match, grow our array (if necessary), and then
                 * add the virtual address of the matching cache to our list.
                 */
                if (umc->umc_nelems >= umc->umc_size) {
                        s = umc->umc_size ? umc->umc_size * 2 : 256;
                        p = mdb_alloc(sizeof (uintptr_t) * s, UM_SLEEP | UM_GC);

                        bcopy(umc->umc_caches, p,
                            sizeof (uintptr_t) * umc->umc_size);

                        umc->umc_caches = p;
                        umc->umc_size = s;
                }

                umc->umc_caches[umc->umc_nelems++] = addr;
                return (umc->umc_name ? WALK_DONE : WALK_NEXT);
        }

        return (WALK_NEXT);
}

/*
 * The second piece of ::umausers is a hash table of allocations.  Each
 * allocation owner is identified by its stack trace and data_size.  We then
 * track the total bytes of all such allocations, and the number of allocations
 * to report at the end.  Once we have a list of caches, we walk through the
 * allocated bufctls of each, and update our hash table accordingly.
 */

typedef struct umowner {
        struct umowner *umo_head;               /* First hash elt in bucket */
        struct umowner *umo_next;               /* Next hash elt in chain */
        size_t umo_signature;                   /* Hash table signature */
        uint_t umo_num;                         /* Number of allocations */
        size_t umo_data_size;                   /* Size of each allocation */
        size_t umo_total_size;                  /* Total bytes of allocation */
        int umo_depth;                          /* Depth of stack trace */
        uintptr_t *umo_stack;                   /* Stack trace */
} umowner_t;

typedef struct umusers {
        const umem_cache_t *umu_cache;          /* Current umem cache */
        umowner_t *umu_hash;                    /* Hash table of owners */
        uintptr_t *umu_stacks;                  /* stacks for owners */
        int umu_nelems;                         /* Number of entries in use */
        int umu_size;                           /* Total number of entries */
} umusers_t;

static void
umu_add(umusers_t *umu, const umem_bufctl_audit_t *bcp,
    size_t size, size_t data_size)
{
        int i, depth = MIN(bcp->bc_depth, umem_stack_depth);
        size_t bucket, signature = data_size;
        umowner_t *umo, *umoend;

        /*
         * If the hash table is full, double its size and rehash everything.
         */
        if (umu->umu_nelems >= umu->umu_size) {
                int s = umu->umu_size ? umu->umu_size * 2 : 1024;
                size_t umowner_size = sizeof (umowner_t);
                size_t trace_size = umem_stack_depth * sizeof (uintptr_t);
                uintptr_t *new_stacks;

                umo = mdb_alloc(umowner_size * s, UM_SLEEP | UM_GC);
                new_stacks = mdb_alloc(trace_size * s, UM_SLEEP | UM_GC);

                bcopy(umu->umu_hash, umo, umowner_size * umu->umu_size);
                bcopy(umu->umu_stacks, new_stacks, trace_size * umu->umu_size);
                umu->umu_hash = umo;
                umu->umu_stacks = new_stacks;
                umu->umu_size = s;

                umoend = umu->umu_hash + umu->umu_size;
                for (umo = umu->umu_hash; umo < umoend; umo++) {
                        umo->umo_head = NULL;
                        umo->umo_stack = &umu->umu_stacks[
                            umem_stack_depth * (umo - umu->umu_hash)];
                }

                umoend = umu->umu_hash + umu->umu_nelems;
                for (umo = umu->umu_hash; umo < umoend; umo++) {
                        bucket = umo->umo_signature & (umu->umu_size - 1);
                        umo->umo_next = umu->umu_hash[bucket].umo_head;
                        umu->umu_hash[bucket].umo_head = umo;
                }
        }

        /*
         * Finish computing the hash signature from the stack trace, and then
         * see if the owner is in the hash table.  If so, update our stats.
         */
        for (i = 0; i < depth; i++)
                signature += bcp->bc_stack[i];

        bucket = signature & (umu->umu_size - 1);

        for (umo = umu->umu_hash[bucket].umo_head; umo; umo = umo->umo_next) {
                if (umo->umo_signature == signature) {
                        size_t difference = 0;

                        difference |= umo->umo_data_size - data_size;
                        difference |= umo->umo_depth - depth;

                        for (i = 0; i < depth; i++) {
                                difference |= umo->umo_stack[i] -
                                    bcp->bc_stack[i];
                        }

                        if (difference == 0) {
                                umo->umo_total_size += size;
                                umo->umo_num++;
                                return;
                        }
                }
        }

        /*
         * If the owner is not yet hashed, grab the next element and fill it
         * in based on the allocation information.
         */
        umo = &umu->umu_hash[umu->umu_nelems++];
        umo->umo_next = umu->umu_hash[bucket].umo_head;
        umu->umu_hash[bucket].umo_head = umo;

        umo->umo_signature = signature;
        umo->umo_num = 1;
        umo->umo_data_size = data_size;
        umo->umo_total_size = size;
        umo->umo_depth = depth;

        for (i = 0; i < depth; i++)
                umo->umo_stack[i] = bcp->bc_stack[i];
}

/*
 * When ::umausers is invoked without the -f flag, we simply update our hash
 * table with the information from each allocated bufctl.
 */
/*ARGSUSED*/
static int
umause1(uintptr_t addr, const umem_bufctl_audit_t *bcp, umusers_t *umu)
{
        const umem_cache_t *cp = umu->umu_cache;

        umu_add(umu, bcp, cp->cache_bufsize, cp->cache_bufsize);
        return (WALK_NEXT);
}

/*
 * When ::umausers is invoked with the -f flag, we print out the information
 * for each bufctl as well as updating the hash table.
 */
static int
umause2(uintptr_t addr, const umem_bufctl_audit_t *bcp, umusers_t *umu)
{
        int i, depth = MIN(bcp->bc_depth, umem_stack_depth);
        const umem_cache_t *cp = umu->umu_cache;

        mdb_printf("size %d, addr %p, thread %p, cache %s\n",
            cp->cache_bufsize, addr, bcp->bc_thread, cp->cache_name);

        for (i = 0; i < depth; i++)
                mdb_printf("\t %a\n", bcp->bc_stack[i]);

        umu_add(umu, bcp, cp->cache_bufsize, cp->cache_bufsize);
        return (WALK_NEXT);
}

/*
 * We sort our results by allocation size before printing them.
 */
static int
umownercmp(const void *lp, const void *rp)
{
        const umowner_t *lhs = lp;
        const umowner_t *rhs = rp;

        return (rhs->umo_total_size - lhs->umo_total_size);
}

/*
 * The main engine of ::umausers is relatively straightforward: First we
 * accumulate our list of umem_cache_t addresses into the umclist_t. Next we
 * iterate over the allocated bufctls of each cache in the list.  Finally,
 * we sort and print our results.
 */
/*ARGSUSED*/
int
umausers(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
        int mem_threshold = 8192;       /* Minimum # bytes for printing */
        int cnt_threshold = 100;        /* Minimum # blocks for printing */
        int audited_caches = 0;         /* Number of UMF_AUDIT caches found */
        int do_all_caches = 1;          /* Do all caches (no arguments) */
        int opt_e = FALSE;              /* Include "small" users */
        int opt_f = FALSE;              /* Print stack traces */

        mdb_walk_cb_t callback = (mdb_walk_cb_t)umause1;
        umowner_t *umo, *umoend;
        int i, oelems;

        umclist_t umc;
        umusers_t umu;

        if (flags & DCMD_ADDRSPEC)
                return (DCMD_USAGE);

        bzero(&umc, sizeof (umc));
        bzero(&umu, sizeof (umu));

        while ((i = mdb_getopts(argc, argv,
            'e', MDB_OPT_SETBITS, TRUE, &opt_e,
            'f', MDB_OPT_SETBITS, TRUE, &opt_f, NULL)) != argc) {

                argv += i;      /* skip past options we just processed */
                argc -= i;      /* adjust argc */

                if (argv->a_type != MDB_TYPE_STRING || *argv->a_un.a_str == '-')
                        return (DCMD_USAGE);

                oelems = umc.umc_nelems;
                umc.umc_name = argv->a_un.a_str;
                (void) mdb_walk("umem_cache", (mdb_walk_cb_t)umc_add, &umc);

                if (umc.umc_nelems == oelems) {
                        mdb_warn("unknown umem cache: %s\n", umc.umc_name);
                        return (DCMD_ERR);
                }

                do_all_caches = 0;
                argv++;
                argc--;
        }

        if (opt_e)
                mem_threshold = cnt_threshold = 0;

        if (opt_f)
                callback = (mdb_walk_cb_t)umause2;

        if (do_all_caches) {
                umc.umc_name = NULL; /* match all cache names */
                (void) mdb_walk("umem_cache", (mdb_walk_cb_t)umc_add, &umc);
        }

        for (i = 0; i < umc.umc_nelems; i++) {
                uintptr_t cp = umc.umc_caches[i];
                umem_cache_t c;

                if (mdb_vread(&c, sizeof (c), cp) == -1) {
                        mdb_warn("failed to read cache at %p", cp);
                        continue;
                }

                if (!(c.cache_flags & UMF_AUDIT)) {
                        if (!do_all_caches) {
                                mdb_warn("UMF_AUDIT is not enabled for %s\n",
                                    c.cache_name);
                        }
                        continue;
                }

                umu.umu_cache = &c;
                (void) mdb_pwalk("bufctl", callback, &umu, cp);
                audited_caches++;
        }

        if (audited_caches == 0 && do_all_caches) {
                mdb_warn("UMF_AUDIT is not enabled for any caches\n");
                return (DCMD_ERR);
        }

        qsort(umu.umu_hash, umu.umu_nelems, sizeof (umowner_t), umownercmp);
        umoend = umu.umu_hash + umu.umu_nelems;

        for (umo = umu.umu_hash; umo < umoend; umo++) {
                if (umo->umo_total_size < mem_threshold &&
                    umo->umo_num < cnt_threshold)
                        continue;
                mdb_printf("%lu bytes for %u allocations with data size %lu:\n",
                    umo->umo_total_size, umo->umo_num, umo->umo_data_size);
                for (i = 0; i < umo->umo_depth; i++)
                        mdb_printf("\t %a\n", umo->umo_stack[i]);
        }

        return (DCMD_OK);
}

struct malloc_data {
        uint32_t malloc_size;
        uint32_t malloc_stat; /* == UMEM_MALLOC_ENCODE(state, malloc_size) */
};

#ifdef _LP64
#define UMI_MAX_BUCKET          (UMEM_MAXBUF - 2*sizeof (struct malloc_data))
#else
#define UMI_MAX_BUCKET          (UMEM_MAXBUF - sizeof (struct malloc_data))
#endif

typedef struct umem_malloc_info {
        size_t um_total;        /* total allocated buffers */
        size_t um_malloc;       /* malloc buffers */
        size_t um_malloc_size;  /* sum of malloc buffer sizes */
        size_t um_malloc_overhead; /* sum of in-chunk overheads */

        umem_cache_t *um_cp;

        uint_t *um_bucket;
} umem_malloc_info_t;

static void
umem_malloc_print_dist(uint_t *um_bucket, size_t minmalloc, size_t maxmalloc,
    size_t maxbuckets, size_t minbucketsize, int geometric)
{
        uint64_t um_malloc;
        int minb = -1;
        int maxb = -1;
        int buckets;
        int nbucks;
        int i;
        int b;
        const int *distarray;

        minb = (int)minmalloc;
        maxb = (int)maxmalloc;

        nbucks = buckets = maxb - minb + 1;

        um_malloc = 0;
        for (b = minb; b <= maxb; b++)
                um_malloc += um_bucket[b];

        if (maxbuckets != 0)
                buckets = MIN(buckets, maxbuckets);

        if (minbucketsize > 1) {
                buckets = MIN(buckets, nbucks/minbucketsize);
                if (buckets == 0) {
                        buckets = 1;
                        minbucketsize = nbucks;
                }
        }

        if (geometric)
                distarray = dist_geometric(buckets, minb, maxb, minbucketsize);
        else
                distarray = dist_linear(buckets, minb, maxb);

        dist_print_header("malloc size", 11, "count");
        for (i = 0; i < buckets; i++) {
                dist_print_bucket(distarray, i, um_bucket, um_malloc, 11);
        }
        mdb_printf("\n");
}

/*
 * A malloc()ed buffer looks like:
 *
 *      <----------- mi.malloc_size --->
 *      <----------- cp.cache_bufsize ------------------>
 *      <----------- cp.cache_chunksize -------------------------------->
 *      +-------+-----------------------+---------------+---------------+
 *      |/tag///| mallocsz              |/round-off/////|/debug info////|
 *      +-------+---------------------------------------+---------------+
 *              <-- usable space ------>
 *
 * mallocsz is the argument to malloc(3C).
 * mi.malloc_size is the actual size passed to umem_alloc(), which
 * is rounded up to the smallest available cache size, which is
 * cache_bufsize.  If there is debugging or alignment overhead in
 * the cache, that is reflected in a larger cache_chunksize.
 *
 * The tag at the beginning of the buffer is either 8-bytes or 16-bytes,
 * depending upon the ISA's alignment requirements.  For 32-bit allocations,
 * it is always a 8-byte tag.  For 64-bit allocations larger than 8 bytes,
 * the tag has 8 bytes of padding before it.
 *
 * 32-byte, 64-byte buffers <= 8 bytes:
 *      +-------+-------+--------- ...
 *      |/size//|/stat//| mallocsz ...
 *      +-------+-------+--------- ...
 *                      ^
 *                      pointer returned from malloc(3C)
 *
 * 64-byte buffers > 8 bytes:
 *      +---------------+-------+-------+--------- ...
 *      |/padding///////|/size//|/stat//| mallocsz ...
 *      +---------------+-------+-------+--------- ...
 *                                      ^
 *                                      pointer returned from malloc(3C)
 *
 * The "size" field is "malloc_size", which is mallocsz + the padding.
 * The "stat" field is derived from malloc_size, and functions as a
 * validation that this buffer is actually from malloc(3C).
 */
/*ARGSUSED*/
static int
um_umem_buffer_cb(uintptr_t addr, void *buf, umem_malloc_info_t *ump)
{
        struct malloc_data md;
        size_t m_addr = addr;
        size_t overhead = sizeof (md);
        size_t mallocsz;

        ump->um_total++;

#ifdef _LP64
        if (ump->um_cp->cache_bufsize > UMEM_SECOND_ALIGN) {
                m_addr += overhead;
                overhead += sizeof (md);
        }
#endif

        if (mdb_vread(&md, sizeof (md), m_addr) == -1) {
                mdb_warn("unable to read malloc header at %p", m_addr);
                return (WALK_NEXT);
        }

        switch (UMEM_MALLOC_DECODE(md.malloc_stat, md.malloc_size)) {
        case MALLOC_MAGIC:
#ifdef _LP64
        case MALLOC_SECOND_MAGIC:
#endif
                mallocsz = md.malloc_size - overhead;

                ump->um_malloc++;
                ump->um_malloc_size += mallocsz;
                ump->um_malloc_overhead += overhead;

                /* include round-off and debug overhead */
                ump->um_malloc_overhead +=
                    ump->um_cp->cache_chunksize - md.malloc_size;

                if (ump->um_bucket != NULL && mallocsz <= UMI_MAX_BUCKET)
                        ump->um_bucket[mallocsz]++;

                break;
        default:
                break;
        }

        return (WALK_NEXT);
}

int
get_umem_alloc_sizes(int **out, size_t *out_num)
{
        GElf_Sym sym;

        if (umem_lookup_by_name("umem_alloc_sizes", &sym) == -1) {
                mdb_warn("unable to look up umem_alloc_sizes");
                return (-1);
        }

        *out = mdb_alloc(sym.st_size, UM_SLEEP | UM_GC);
        *out_num = sym.st_size / sizeof (int);

        if (mdb_vread(*out, sym.st_size, sym.st_value) == -1) {
                mdb_warn("unable to read umem_alloc_sizes (%p)", sym.st_value);
                *out = NULL;
                return (-1);
        }

        return (0);
}


static int
um_umem_cache_cb(uintptr_t addr, umem_cache_t *cp, umem_malloc_info_t *ump)
{
        if (strncmp(cp->cache_name, "umem_alloc_", strlen("umem_alloc_")) != 0)
                return (WALK_NEXT);

        ump->um_cp = cp;

        if (mdb_pwalk("umem", (mdb_walk_cb_t)um_umem_buffer_cb, ump, addr) ==
            -1) {
                mdb_warn("can't walk 'umem' for cache %p", addr);
                return (WALK_ERR);
        }

        return (WALK_NEXT);
}

void
umem_malloc_dist_help(void)
{
        mdb_printf("%s\n",
            "report distribution of outstanding malloc()s");
        mdb_dec_indent(2);
        mdb_printf("%<b>OPTIONS%</b>\n");
        mdb_inc_indent(2);
        mdb_printf("%s",
"  -b maxbins\n"
"        Use at most maxbins bins for the data\n"
"  -B minbinsize\n"
"        Make the bins at least minbinsize bytes apart\n"
"  -d    dump the raw data out, without binning\n"
"  -g    use geometric binning instead of linear binning\n");
}

/*ARGSUSED*/
int
umem_malloc_dist(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
        umem_malloc_info_t mi;
        uint_t geometric = 0;
        uint_t dump = 0;
        size_t maxbuckets = 0;
        size_t minbucketsize = 0;

        size_t minalloc = 0;
        size_t maxalloc = UMI_MAX_BUCKET;

        if (flags & DCMD_ADDRSPEC)
                return (DCMD_USAGE);

        if (mdb_getopts(argc, argv,
            'd', MDB_OPT_SETBITS, TRUE, &dump,
            'g', MDB_OPT_SETBITS, TRUE, &geometric,
            'b', MDB_OPT_UINTPTR, &maxbuckets,
            'B', MDB_OPT_UINTPTR, &minbucketsize,
            NULL) != argc)
                return (DCMD_USAGE);

        bzero(&mi, sizeof (mi));
        mi.um_bucket = mdb_zalloc((UMI_MAX_BUCKET + 1) * sizeof (*mi.um_bucket),
            UM_SLEEP | UM_GC);

        if (mdb_walk("umem_cache", (mdb_walk_cb_t)um_umem_cache_cb,
            &mi) == -1) {
                mdb_warn("unable to walk 'umem_cache'");
                return (DCMD_ERR);
        }

        if (dump) {
                int i;
                for (i = minalloc; i <= maxalloc; i++)
                        mdb_printf("%d\t%d\n", i, mi.um_bucket[i]);

                return (DCMD_OK);
        }

        umem_malloc_print_dist(mi.um_bucket, minalloc, maxalloc,
            maxbuckets, minbucketsize, geometric);

        return (DCMD_OK);
}

void
umem_malloc_info_help(void)
{
        mdb_printf("%s\n",
            "report information about malloc()s by cache.  ");
        mdb_dec_indent(2);
        mdb_printf("%<b>OPTIONS%</b>\n");
        mdb_inc_indent(2);
        mdb_printf("%s",
"  -b maxbins\n"
"        Use at most maxbins bins for the data\n"
"  -B minbinsize\n"
"        Make the bins at least minbinsize bytes apart\n"
"  -d    dump the raw distribution data without binning\n"
#ifndef _KMDB
"  -g    use geometric binning instead of linear binning\n"
#endif
            "");
}
int
umem_malloc_info(uintptr_t addr, uint_t flags, int argc, const mdb_arg_t *argv)
{
        umem_cache_t c;
        umem_malloc_info_t mi;

        int skip = 0;

        size_t maxmalloc;
        size_t overhead;
        size_t allocated;
        size_t avg_malloc;
        size_t overhead_pct;    /* 1000 * overhead_percent */

        uint_t verbose = 0;
        uint_t dump = 0;
        uint_t geometric = 0;
        size_t maxbuckets = 0;
        size_t minbucketsize = 0;

        int *alloc_sizes;
        int idx;
        size_t num;
        size_t minmalloc;

        if (mdb_getopts(argc, argv,
            'd', MDB_OPT_SETBITS, TRUE, &dump,
            'g', MDB_OPT_SETBITS, TRUE, &geometric,
            'b', MDB_OPT_UINTPTR, &maxbuckets,
            'B', MDB_OPT_UINTPTR, &minbucketsize,
            NULL) != argc)
                return (DCMD_USAGE);

        if (dump || geometric || (maxbuckets != 0) || (minbucketsize != 0))
                verbose = 1;

        if (!(flags & DCMD_ADDRSPEC)) {
                if (mdb_walk_dcmd("umem_cache", "umem_malloc_info",
                    argc, argv) == -1) {
                        mdb_warn("can't walk umem_cache");
                        return (DCMD_ERR);
                }
                return (DCMD_OK);
        }

        if (!mdb_vread(&c, sizeof (c), addr)) {
                mdb_warn("unable to read cache at %p", addr);
                return (DCMD_ERR);
        }

        if (strncmp(c.cache_name, "umem_alloc_", strlen("umem_alloc_")) != 0) {
                if (!(flags & DCMD_LOOP))
                        mdb_warn("umem_malloc_info: cache \"%s\" is not used "
                            "by malloc()\n", c.cache_name);
                skip = 1;
        }

        /*
         * normally, print the header only the first time.  In verbose mode,
         * print the header on every non-skipped buffer
         */
        if ((!verbose && DCMD_HDRSPEC(flags)) || (verbose && !skip))
                mdb_printf("%<ul>%-?s %6s %6s %8s %8s %10s %10s %6s%</ul>\n",
                    "CACHE", "BUFSZ", "MAXMAL",
                    "BUFMALLC", "AVG_MAL", "MALLOCED", "OVERHEAD", "%OVER");

        if (skip)
                return (DCMD_OK);

        maxmalloc = c.cache_bufsize - sizeof (struct malloc_data);
#ifdef _LP64
        if (c.cache_bufsize > UMEM_SECOND_ALIGN)
                maxmalloc -= sizeof (struct malloc_data);
#endif

        bzero(&mi, sizeof (mi));
        mi.um_cp = &c;
        if (verbose)
                mi.um_bucket =
                    mdb_zalloc((UMI_MAX_BUCKET + 1) * sizeof (*mi.um_bucket),
                    UM_SLEEP | UM_GC);

        if (mdb_pwalk("umem", (mdb_walk_cb_t)um_umem_buffer_cb, &mi, addr) ==
            -1) {
                mdb_warn("can't walk 'umem'");
                return (DCMD_ERR);
        }

        overhead = mi.um_malloc_overhead;
        allocated = mi.um_malloc_size;

        /* do integer round off for the average */
        if (mi.um_malloc != 0)
                avg_malloc = (allocated + (mi.um_malloc - 1)/2) / mi.um_malloc;
        else
                avg_malloc = 0;

        /*
         * include per-slab overhead
         *
         * Each slab in a given cache is the same size, and has the same
         * number of chunks in it;  we read in the first slab on the
         * slab list to get the number of chunks for all slabs.  To
         * compute the per-slab overhead, we just subtract the chunk usage
         * from the slabsize:
         *
         * +------------+-------+-------+ ... --+-------+-------+-------+
         * |////////////|       |       | ...   |       |///////|///////|
         * |////color///| chunk | chunk | ...   | chunk |/color/|/slab//|
         * |////////////|       |       | ...   |       |///////|///////|
         * +------------+-------+-------+ ... --+-------+-------+-------+
         * |            \_______chunksize * chunks_____/                |
         * \__________________________slabsize__________________________/
         *
         * For UMF_HASH caches, there is an additional source of overhead;
         * the external umem_slab_t and per-chunk bufctl structures.  We
         * include those in our per-slab overhead.
         *
         * Once we have a number for the per-slab overhead, we estimate
         * the actual overhead by treating the malloc()ed buffers as if
         * they were densely packed:
         *
         *      additional overhead = (# mallocs) * (per-slab) / (chunks);
         *
         * carefully ordering the multiply before the divide, to avoid
         * round-off error.
         */
        if (mi.um_malloc != 0) {
                umem_slab_t slab;
                uintptr_t saddr = (uintptr_t)c.cache_nullslab.slab_next;

                if (mdb_vread(&slab, sizeof (slab), saddr) == -1) {
                        mdb_warn("unable to read slab at %p\n", saddr);
                } else {
                        long chunks = slab.slab_chunks;
                        if (chunks != 0 && c.cache_chunksize != 0 &&
                            chunks <= c.cache_slabsize / c.cache_chunksize) {
                                uintmax_t perslab =
                                    c.cache_slabsize -
                                    (c.cache_chunksize * chunks);

                                if (c.cache_flags & UMF_HASH) {
                                        perslab += sizeof (umem_slab_t) +
                                            chunks *
                                            ((c.cache_flags & UMF_AUDIT) ?
                                            sizeof (umem_bufctl_audit_t) :
                                            sizeof (umem_bufctl_t));
                                }
                                overhead +=
                                    (perslab * (uintmax_t)mi.um_malloc)/chunks;
                        } else {
                                mdb_warn("invalid #chunks (%d) in slab %p\n",
                                    chunks, saddr);
                        }
                }
        }

        if (allocated != 0)
                overhead_pct = (1000ULL * overhead) / allocated;
        else
                overhead_pct = 0;

        mdb_printf("%0?p %6ld %6ld %8ld %8ld %10ld %10ld %3ld.%01ld%%\n",
            addr, c.cache_bufsize, maxmalloc,
            mi.um_malloc, avg_malloc, allocated, overhead,
            overhead_pct / 10, overhead_pct % 10);

        if (!verbose)
                return (DCMD_OK);

        if (!dump)
                mdb_printf("\n");

        if (get_umem_alloc_sizes(&alloc_sizes, &num) == -1)
                return (DCMD_ERR);

        for (idx = 0; idx < num; idx++) {
                if (alloc_sizes[idx] == c.cache_bufsize)
                        break;
                if (alloc_sizes[idx] == 0) {
                        idx = num;      /* 0-terminated array */
                        break;
                }
        }
        if (idx == num) {
                mdb_warn(
                    "cache %p's size (%d) not in umem_alloc_sizes\n",
                    addr, c.cache_bufsize);
                return (DCMD_ERR);
        }

        minmalloc = (idx == 0)? 0 : alloc_sizes[idx - 1];
        if (minmalloc > 0) {
#ifdef _LP64
                if (minmalloc > UMEM_SECOND_ALIGN)
                        minmalloc -= sizeof (struct malloc_data);
#endif
                minmalloc -= sizeof (struct malloc_data);
                minmalloc += 1;
        }

        if (dump) {
                for (idx = minmalloc; idx <= maxmalloc; idx++)
                        mdb_printf("%d\t%d\n", idx, mi.um_bucket[idx]);
                mdb_printf("\n");
        } else {
                umem_malloc_print_dist(mi.um_bucket, minmalloc, maxmalloc,
                    maxbuckets, minbucketsize, geometric);
        }

        return (DCMD_OK);
}