/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */


#include <sys/errno.h>
#include <sys/stat.h>
#include <sys/modctl.h>
#include <sys/conf.h>
#include <sys/systm.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/cpuvar.h>
#include <sys/kmem.h>
#include <sys/strsubr.h>
#include <sys/dtrace.h>
#include <sys/kobj.h>
#include <sys/modctl.h>
#include <sys/atomic.h>
#include <vm/seg_kmem.h>
#include <sys/stack.h>
#include <sys/ctf_api.h>
#include <sys/sysmacros.h>

static dev_info_t               *fbt_devi;
static dtrace_provider_id_t     fbt_id;
static uintptr_t                fbt_trampoline;
static caddr_t                  fbt_trampoline_window;
static size_t                   fbt_trampoline_size;
static int                      fbt_verbose = 0;

/*
 * Various interesting bean counters.
 */
static int                      fbt_entry;
static int                      fbt_ret;
static int                      fbt_retl;
static int                      fbt_retl_jmptab;
static int                      fbt_retl_twoinstr;
static int                      fbt_retl_tailcall;
static int                      fbt_retl_tailjmpl;
static int                      fbt_leaf_functions;

extern char                     stubs_base[];
extern char                     stubs_end[];

#define FBT_REG_G0              0
#define FBT_REG_G1              1
#define FBT_REG_O0              8
#define FBT_REG_O1              9
#define FBT_REG_O2              10
#define FBT_REG_O3              11
#define FBT_REG_O4              12
#define FBT_REG_O5              13
#define FBT_REG_O6              14
#define FBT_REG_O7              15
#define FBT_REG_I0              24
#define FBT_REG_I1              25
#define FBT_REG_I2              26
#define FBT_REG_I3              27
#define FBT_REG_I4              28
#define FBT_REG_I7              31
#define FBT_REG_L0              16
#define FBT_REG_L1              17
#define FBT_REG_L2              18
#define FBT_REG_L3              19
#define FBT_REG_PC              5

#define FBT_REG_ISGLOBAL(r)     ((r) < 8)
#define FBT_REG_ISOUTPUT(r)     ((r) >= 8 && (r) < 16)
#define FBT_REG_ISLOCAL(r)      ((r) >= 16 && (r) < 24)
#define FBT_REG_ISVOLATILE(r)   \
        ((FBT_REG_ISGLOBAL(r) || FBT_REG_ISOUTPUT(r)) && (r) != FBT_REG_G0)
#define FBT_REG_NLOCALS         8

#define FBT_REG_MARKLOCAL(locals, r)    \
        if (FBT_REG_ISLOCAL(r)) \
                (locals)[(r) - FBT_REG_L0] = 1;

#define FBT_REG_INITLOCALS(local, locals)       \
        for ((local) = 0; (local) < FBT_REG_NLOCALS; (local)++)  \
                (locals)[(local)] = 0; \
        (local) = FBT_REG_L0

#define FBT_REG_ALLOCLOCAL(local, locals)       \
        while ((locals)[(local) - FBT_REG_L0]) \
                (local)++; \
        (locals)[(local) - FBT_REG_L0] = 1;

#define FBT_OP_MASK             0xc0000000
#define FBT_OP_SHIFT            30
#define FBT_OP(val)             ((val) & FBT_FMT1_MASK)

#define FBT_SIMM13_MASK         0x1fff
#define FBT_SIMM13_MAX          ((int32_t)0xfff)
#define FBT_IMM22_MASK          0x3fffff
#define FBT_IMM22_SHIFT         10
#define FBT_IMM10_MASK          0x3ff

#define FBT_DISP30_MASK         0x3fffffff
#define FBT_DISP30(from, to)    \
        (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP30_MASK)

#define FBT_DISP22_MASK         0x3fffff
#define FBT_DISP22(from, to)    \
        (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP22_MASK)

#define FBT_DISP19_MASK         0x7ffff
#define FBT_DISP19(from, to)    \
        (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP19_MASK)

#define FBT_DISP16_HISHIFT      20
#define FBT_DISP16_HIMASK       (0x3 << FBT_DISP16_HISHIFT)
#define FBT_DISP16_LOMASK       (0x3fff)
#define FBT_DISP16_MASK         (FBT_DISP16_HIMASK | FBT_DISP16_LOMASK)
#define FBT_DISP16(val) \
        ((((val) & FBT_DISP16_HIMASK) >> 6) | ((val) & FBT_DISP16_LOMASK))

#define FBT_DISP14_MASK         0x3fff
#define FBT_DISP14(from, to)    \
        (((uintptr_t)(to) - (uintptr_t)(from) >> 2) & FBT_DISP14_MASK)

#define FBT_OP0                 (((uint32_t)0) << FBT_OP_SHIFT)
#define FBT_OP1                 (((uint32_t)1) << FBT_OP_SHIFT)
#define FBT_OP2                 (((uint32_t)2) << FBT_OP_SHIFT)
#define FBT_ILLTRAP             0

#define FBT_ANNUL_SHIFT         29
#define FBT_ANNUL               (1 << FBT_ANNUL_SHIFT)

#define FBT_FMT3_OP3_SHIFT      19
#define FBT_FMT3_OP_MASK        0xc1f80000
#define FBT_FMT3_OP(val)        ((val) & FBT_FMT3_OP_MASK)

#define FBT_FMT3_RD_SHIFT       25
#define FBT_FMT3_RD_MASK        (0x1f << FBT_FMT3_RD_SHIFT)
#define FBT_FMT3_RD(val)        \
        (((val) & FBT_FMT3_RD_MASK) >> FBT_FMT3_RD_SHIFT)

#define FBT_FMT3_RS1_SHIFT      14
#define FBT_FMT3_RS1_MASK       (0x1f << FBT_FMT3_RS1_SHIFT)
#define FBT_FMT3_RS1(val)       \
        (((val) & FBT_FMT3_RS1_MASK) >> FBT_FMT3_RS1_SHIFT)
#define FBT_FMT3_RS1_SET(val, rs1) \
        (val) = ((val) & ~FBT_FMT3_RS1_MASK) | ((rs1) << FBT_FMT3_RS1_SHIFT)

#define FBT_FMT3_RS2_SHIFT      0
#define FBT_FMT3_RS2_MASK       (0x1f << FBT_FMT3_RS2_SHIFT)
#define FBT_FMT3_RS2(val)       \
        (((val) & FBT_FMT3_RS2_MASK) >> FBT_FMT3_RS2_SHIFT)
#define FBT_FMT3_RS2_SET(val, rs2) \
        (val) = ((val) & ~FBT_FMT3_RS2_MASK) | ((rs2) << FBT_FMT3_RS2_SHIFT)

#define FBT_FMT3_IMM_SHIFT      13
#define FBT_FMT3_IMM            (1 << FBT_FMT3_IMM_SHIFT)
#define FBT_FMT3_SIMM13_MASK    FBT_SIMM13_MASK

#define FBT_FMT3_ISIMM(val)     ((val) & FBT_FMT3_IMM)
#define FBT_FMT3_SIMM13(val)    ((val) & FBT_FMT3_SIMM13_MASK)

#define FBT_FMT2_OP2_SHIFT      22
#define FBT_FMT2_OP2_MASK       (0x7 << FBT_FMT2_OP2_SHIFT)
#define FBT_FMT2_RD_SHIFT       25

#define FBT_FMT1_OP(val)        ((val) & FBT_OP_MASK)
#define FBT_FMT1_DISP30(val)    ((val) & FBT_DISP30_MASK)

#define FBT_FMT2_OP2_BPCC       (0x01 << FBT_FMT2_OP2_SHIFT)
#define FBT_FMT2_OP2_BCC        (0x02 << FBT_FMT2_OP2_SHIFT)
#define FBT_FMT2_OP2_BPR        (0x03 << FBT_FMT2_OP2_SHIFT)
#define FBT_FMT2_OP2_SETHI      (0x04 << FBT_FMT2_OP2_SHIFT)

#define FBT_FMT2_COND_SHIFT     25
#define FBT_FMT2_COND_BA        (0x8 << FBT_FMT2_COND_SHIFT)
#define FBT_FMT2_COND_BL        (0x3 << FBT_FMT2_COND_SHIFT)
#define FBT_FMT2_COND_BGE       (0xb << FBT_FMT2_COND_SHIFT)

#define FBT_OP_RESTORE          (FBT_OP2 | (0x3d << FBT_FMT3_OP3_SHIFT))
#define FBT_OP_SAVE             (FBT_OP2 | (0x3c << FBT_FMT3_OP3_SHIFT))
#define FBT_OP_JMPL             (FBT_OP2 | (0x38 << FBT_FMT3_OP3_SHIFT))
#define FBT_OP_RETURN           (FBT_OP2 | (0x39 << FBT_FMT3_OP3_SHIFT))
#define FBT_OP_CALL             FBT_OP1
#define FBT_OP_SETHI            (FBT_OP0 | FBT_FMT2_OP2_SETHI)
#define FBT_OP_ADD              (FBT_OP2 | (0x00 << FBT_FMT3_OP3_SHIFT))
#define FBT_OP_OR               (FBT_OP2 | (0x02 << FBT_FMT3_OP3_SHIFT))
#define FBT_OP_SUB              (FBT_OP2 | (0x04 << FBT_FMT3_OP3_SHIFT))
#define FBT_OP_CC               (FBT_OP2 | (0x10 << FBT_FMT3_OP3_SHIFT))
#define FBT_OP_BA               (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BA)
#define FBT_OP_BL               (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BL)
#define FBT_OP_BGE              (FBT_OP0 | FBT_FMT2_OP2_BCC | FBT_FMT2_COND_BGE)
#define FBT_OP_BAPCC            (FBT_OP0 | FBT_FMT2_OP2_BPCC | FBT_FMT2_COND_BA)
#define FBT_OP_RD               (FBT_OP2 | (0x28 << FBT_FMT3_OP3_SHIFT))

#define FBT_ORLO(rs, val, rd) \
        (FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \
        ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_IMM10_MASK))

#define FBT_ORSIMM13(rs, val, rd) \
        (FBT_OP_OR | ((rs) << FBT_FMT3_RS1_SHIFT) | \
        ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))

#define FBT_ADDSIMM13(rs, val, rd) \
        (FBT_OP_ADD | ((rs) << FBT_FMT3_RS1_SHIFT) | \
        ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))

#define FBT_ADD(rs1, rs2, rd) \
        (FBT_OP_ADD | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
        ((rs2) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT))

#define FBT_CMP(rs1, rs2) \
        (FBT_OP_SUB | FBT_OP_CC | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
        ((rs2) << FBT_FMT3_RS2_SHIFT) | (FBT_REG_G0 << FBT_FMT3_RD_SHIFT))

#define FBT_MOV(rs, rd) \
        (FBT_OP_OR | (FBT_REG_G0 << FBT_FMT3_RS1_SHIFT) | \
        ((rs) << FBT_FMT3_RS2_SHIFT) | ((rd) << FBT_FMT3_RD_SHIFT))

#define FBT_SETHI(val, reg)     \
        (FBT_OP_SETHI | (reg << FBT_FMT2_RD_SHIFT) | \
        ((val >> FBT_IMM22_SHIFT) & FBT_IMM22_MASK))

#define FBT_CALL(orig, dest)    (FBT_OP_CALL | FBT_DISP30(orig, dest))

#define FBT_RET \
        (FBT_OP_JMPL | (FBT_REG_I7 << FBT_FMT3_RS1_SHIFT) | \
        (FBT_REG_G0 << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | (sizeof (pc_t) << 1))

#define FBT_SAVEIMM(rd, val, rs1)       \
        (FBT_OP_SAVE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
        ((rd) << FBT_FMT3_RD_SHIFT) | FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))

#define FBT_RESTORE(rd, rs1, rs2)       \
        (FBT_OP_RESTORE | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
        ((rd) << FBT_FMT3_RD_SHIFT) | ((rs2) << FBT_FMT3_RS2_SHIFT))

#define FBT_RETURN(rs1, val)            \
        (FBT_OP_RETURN | ((rs1) << FBT_FMT3_RS1_SHIFT) | \
        FBT_FMT3_IMM | ((val) & FBT_SIMM13_MASK))

#define FBT_BA(orig, dest)      (FBT_OP_BA | FBT_DISP22(orig, dest))
#define FBT_BAA(orig, dest)     (FBT_BA(orig, dest) | FBT_ANNUL)
#define FBT_BL(orig, dest)      (FBT_OP_BL | FBT_DISP22(orig, dest))
#define FBT_BGE(orig, dest)     (FBT_OP_BGE | FBT_DISP22(orig, dest))
#define FBT_BDEST(va, instr)    ((uintptr_t)(va) + \
        (((int32_t)(((instr) & FBT_DISP22_MASK) << 10)) >> 8))
#define FBT_BPCCDEST(va, instr) ((uintptr_t)(va) + \
        (((int32_t)(((instr) & FBT_DISP19_MASK) << 13)) >> 11))
#define FBT_BPRDEST(va, instr)  ((uintptr_t)(va) + \
        (((int32_t)((FBT_DISP16(instr)) << 16)) >> 14))

/*
 * We're only going to treat a save as safe if (a) both rs1 and rd are
 * %sp and (b) if the instruction has a simm, the value isn't 0.
 */
#define FBT_IS_SAVE(instr)      \
        (FBT_FMT3_OP(instr) == FBT_OP_SAVE && \
        FBT_FMT3_RD(instr) == FBT_REG_O6 && \
        FBT_FMT3_RS1(instr) == FBT_REG_O6 && \
        !(FBT_FMT3_ISIMM(instr) && FBT_FMT3_SIMM13(instr) == 0))

#define FBT_IS_BA(instr)        (((instr) & ~FBT_DISP22_MASK) == FBT_OP_BA)
#define FBT_IS_BAPCC(instr)     (((instr) & ~FBT_DISP22_MASK) == FBT_OP_BAPCC)

#define FBT_IS_RDPC(instr)      ((FBT_FMT3_OP(instr) == FBT_OP_RD) && \
        (FBT_FMT3_RD(instr) == FBT_REG_PC))

#define FBT_IS_PCRELATIVE(instr)        \
        ((((instr) & FBT_OP_MASK) == FBT_OP0 && \
        ((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \
        ((instr) & FBT_OP_MASK) == FBT_OP1 || \
        FBT_IS_RDPC(instr))

#define FBT_IS_CTI(instr)       \
        ((((instr) & FBT_OP_MASK) == FBT_OP0 && \
        ((instr) & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI) || \
        ((instr) & FBT_OP_MASK) == FBT_OP1 || \
        (FBT_FMT3_OP(instr) == FBT_OP_JMPL) || \
        (FBT_FMT3_OP(instr) == FBT_OP_RETURN))

#define FBT_PROBENAME_ENTRY     "entry"
#define FBT_PROBENAME_RETURN    "return"
#define FBT_ESTIMATE_ID         (UINT32_MAX)
#define FBT_COUNTER(id, count)  if ((id) != FBT_ESTIMATE_ID) (count)++

#define FBT_ENTENT_MAXSIZE      (16 * sizeof (uint32_t))
#define FBT_RETENT_MAXSIZE      (11 * sizeof (uint32_t))
#define FBT_RETLENT_MAXSIZE     (23 * sizeof (uint32_t))
#define FBT_ENT_MAXSIZE         \
        MAX(MAX(FBT_ENTENT_MAXSIZE, FBT_RETENT_MAXSIZE), FBT_RETLENT_MAXSIZE)

typedef struct fbt_probe {
        char            *fbtp_name;
        dtrace_id_t     fbtp_id;
        uintptr_t       fbtp_addr;
        struct modctl   *fbtp_ctl;
        int             fbtp_loadcnt;
        int             fbtp_symndx;
        int             fbtp_primary;
        int             fbtp_return;
        uint32_t        *fbtp_patchpoint;
        uint32_t        fbtp_patchval;
        uint32_t        fbtp_savedval;
        struct fbt_probe *fbtp_next;
} fbt_probe_t;

typedef struct fbt_trampoline {
        uintptr_t       fbtt_va;
        uintptr_t       fbtt_limit;
        uintptr_t       fbtt_next;
} fbt_trampoline_t;

static caddr_t
fbt_trampoline_map(uintptr_t tramp, size_t size)
{
        uintptr_t offs;
        page_t **ppl;

        ASSERT(fbt_trampoline_window == NULL);
        ASSERT(fbt_trampoline_size == 0);
        ASSERT(fbt_trampoline == NULL);

        size += tramp & PAGEOFFSET;
        fbt_trampoline = tramp & PAGEMASK;
        fbt_trampoline_size = (size + PAGESIZE - 1) & PAGEMASK;
        fbt_trampoline_window =
            vmem_alloc(heap_arena, fbt_trampoline_size, VM_SLEEP);

        (void) as_pagelock(&kas, &ppl, (caddr_t)fbt_trampoline,
            fbt_trampoline_size, S_WRITE);

        for (offs = 0; offs < fbt_trampoline_size; offs += PAGESIZE) {
                hat_devload(kas.a_hat, fbt_trampoline_window + offs, PAGESIZE,
                    hat_getpfnum(kas.a_hat, (caddr_t)fbt_trampoline + offs),
                    PROT_READ | PROT_WRITE,
                    HAT_LOAD_LOCK | HAT_LOAD_NOCONSIST);
        }

        as_pageunlock(&kas, ppl, (caddr_t)fbt_trampoline, fbt_trampoline_size,
            S_WRITE);

        return (fbt_trampoline_window + (tramp & PAGEOFFSET));
}

static void
fbt_trampoline_unmap()
{
        ASSERT(fbt_trampoline_window != NULL);
        ASSERT(fbt_trampoline_size != 0);
        ASSERT(fbt_trampoline != NULL);

        membar_enter();
        sync_icache((caddr_t)fbt_trampoline, fbt_trampoline_size);
        sync_icache(fbt_trampoline_window, fbt_trampoline_size);

        hat_unload(kas.a_hat, fbt_trampoline_window, fbt_trampoline_size,
            HAT_UNLOAD_UNLOCK);

        vmem_free(heap_arena, fbt_trampoline_window, fbt_trampoline_size);

        fbt_trampoline_window = NULL;
        fbt_trampoline = 0;
        fbt_trampoline_size = 0;
}

static uintptr_t
fbt_patch_entry(uint32_t *instr, uint32_t id, fbt_trampoline_t *tramp,
    int nargs)
{
        uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
        uint32_t first = *instr;
        uintptr_t va = tramp->fbtt_va;
        uintptr_t base = tramp->fbtt_next;

        if (tramp->fbtt_next + FBT_ENTENT_MAXSIZE > tramp->fbtt_limit) {
                /*
                 * There isn't sufficient room for this entry; return failure.
                 */
                return (0);
        }

        FBT_COUNTER(id, fbt_entry);

        if (FBT_IS_SAVE(first)) {
                *tinstr++ = first;
        } else {
                *tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6);
        }

        if (id > (uint32_t)FBT_SIMM13_MAX) {
                *tinstr++ = FBT_SETHI(id, FBT_REG_O0);
                *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
        } else {
                *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
        }

        if (nargs >= 1)
                *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O1);

        if (nargs >= 2)
                *tinstr++ = FBT_MOV(FBT_REG_I1, FBT_REG_O2);

        if (nargs >= 3)
                *tinstr++ = FBT_MOV(FBT_REG_I2, FBT_REG_O3);

        if (nargs >= 4)
                *tinstr++ = FBT_MOV(FBT_REG_I3, FBT_REG_O4);

        if (nargs >= 5)
                *tinstr++ = FBT_MOV(FBT_REG_I4, FBT_REG_O5);

        if (FBT_IS_SAVE(first)) {
                uintptr_t ret = (uintptr_t)instr - sizeof (uint32_t);

                *tinstr++ = FBT_SETHI(ret, FBT_REG_G1);
                *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
                tinstr++;
                *tinstr++ = FBT_ORLO(FBT_REG_G1, ret, FBT_REG_O7);
        } else {
                uintptr_t slot = *--tinstr;
                uintptr_t ret = (uintptr_t)instr + sizeof (uint32_t);
                uint32_t delay = first;

                *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
                tinstr++;
                *tinstr++ = slot;
                *tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);

                if (FBT_IS_BA(first) || FBT_IS_BAPCC(first)) {
                        /*
                         * This is a special case:  we are instrumenting a
                         * a non-annulled branch-always (or variant).  We'll
                         * return directly to the destination of the branch,
                         * copying the instruction in the delay slot here,
                         * and then executing it in the slot of a ba.
                         */
                        if (FBT_IS_BA(first)) {
                                ret = FBT_BDEST(instr, *instr);
                        } else {
                                ret = FBT_BPCCDEST(instr, *instr);
                        }

                        delay = *(instr + 1);
                }

                if ((first & FBT_OP_MASK) != FBT_OP0 ||
                    (first & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) {
                        *tinstr = FBT_BA((uintptr_t)tinstr - base + va, ret);
                        tinstr++;
                        *tinstr++ = delay;
                } else {
                        /*
                         * If this is a branch-on-register, we have a little
                         * more work to do:  because the displacement is only
                         * sixteen bits, we're going to thunk the branch into
                         * the trampoline, and then ba,a to the appropriate
                         * destination in the branch targets.  That is, we're
                         * constructing this sequence in the trampoline:
                         *
                         *              br[cc]  %[rs], 1f
                         *              <delay-instruction>
                         *              ba,a    <not-taken-destination>
                         *      1:      ba,a    <taken-destination>
                         *
                         */
                        uintptr_t targ = FBT_BPRDEST(instr, first);

                        *tinstr = first & ~(FBT_DISP16_MASK);
                        *tinstr |= FBT_DISP14(tinstr, &tinstr[3]);
                        tinstr++;
                        *tinstr++ = *(instr + 1);
                        *tinstr = FBT_BAA((uintptr_t)tinstr - base + va,
                            ret + sizeof (uint32_t));
                        tinstr++;
                        *tinstr = FBT_BAA((uintptr_t)tinstr - base + va, targ);
                        tinstr++;
                }
        }

        tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
        tramp->fbtt_next = (uintptr_t)tinstr;

        return (1);
}

/*
 * We are patching control-transfer/restore couplets.  There are three
 * variants of couplet:
 *
 * (a)  return          rs1 + imm
 *      delay
 *
 * (b)  jmpl            rs1 + (rs2 | offset), rd
 *      restore         rs1, rs2 | imm, rd
 *
 * (c)  call            displacement
 *      restore         rs1, rs2 | imm, rd
 *
 * If rs1 in (a) is anything other than %i7, or imm is anything other than 8,
 * or delay is a DCTI, we fail.  If rd from the jmpl in (b) is something other
 * than %g0 (a ret or a tail-call through a function pointer) or %o7 (a call
 * through a register), we fail.
 *
 * Note that rs1 and rs2 in the restore instructions in (b) and (c) are
 * potentially outputs and/or globals.  Because these registers cannot be
 * relied upon across the call to dtrace_probe(), we move rs1 into an unused
 * local, ls0, and rs2 into an unused local, ls1, and restructure the restore
 * to be:
 *
 *      restore         ls0, ls1, rd
 *
 * Likewise, rs1 and rs2 in the jmpl of case (b) may be outputs and/or globals.
 * If the jmpl uses outputs or globals, we restructure it to be:
 *
 *      jmpl            ls2 + (ls3 | offset), (%g0 | %o7)
 *
 */
/*ARGSUSED*/
static int
fbt_canpatch_return(uint32_t *instr, int offset, const char *name)
{
        int rd;

        if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) {
                uint32_t delay = *(instr + 1);

                if (*instr != FBT_RETURN(FBT_REG_I7, 8)) {
                        /*
                         * It's unclear if we should warn about this or not.
                         * We really wouldn't expect the compiler to generate
                         * return instructions with something other than %i7
                         * as rs1 and 8 as the simm13 -- it would just be
                         * mean-spirited.  That said, such a construct isn't
                         * necessarily incorrect.  Sill, we err on the side of
                         * caution and warn about it...
                         */
                        cmn_err(CE_NOTE, "cannot instrument return of %s at "
                            "%p: non-canonical return instruction", name,
                            (void *)instr);
                        return (0);
                }

                if (FBT_IS_CTI(delay)) {
                        /*
                         * This is even weirder -- a DCTI coupled with a
                         * return instruction.  Similar constructs are used to
                         * return from utraps, but these typically have the
                         * return in the slot -- and we wouldn't expect to see
                         * it in the kernel regardless.  At any rate, we don't
                         * want to try to instrument this construct, whatever
                         * it may be.
                         */
                        cmn_err(CE_NOTE, "cannot instrument return of %s at "
                            "%p: CTI in delay slot of return instruction",
                            name, (void *)instr);
                        return (0);
                }

                if (FBT_IS_PCRELATIVE(delay)) {
                        /*
                         * This is also very weird, but might be correct code
                         * if the function is (for example) returning the
                         * address of the delay instruction of the return as
                         * its return value (e.g. "rd %pc, %o0" in the slot).
                         * Perhaps correct, but still too weird to not warn
                         * about it...
                         */
                        cmn_err(CE_NOTE, "cannot instrument return of %s at "
                            "%p: PC-relative instruction in delay slot of "
                            "return instruction", name, (void *)instr);
                        return (0);
                }

                return (1);
        }

        if (FBT_FMT3_OP(*(instr + 1)) != FBT_OP_RESTORE)
                return (0);

        if (FBT_FMT1_OP(*instr) == FBT_OP_CALL)
                return (1);

        if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL)
                return (0);

        rd = FBT_FMT3_RD(*instr);

        if (rd == FBT_REG_I7 || rd == FBT_REG_O7 || rd == FBT_REG_G0)
                return (1);

        /*
         * We have encountered a jmpl that is storing the calling %pc in
         * some register besides %i7, %o7 or %g0.  This is strange; emit
         * a warning and fail.
         */
        cmn_err(CE_NOTE, "cannot instrument return of %s at %p: unexpected "
            "jmpl destination register", name, (void *)instr);
        return (0);
}

static int
fbt_canpatch_retl(uint32_t *instr, int offset, const char *name)
{
        if (FBT_FMT1_OP(*instr) == FBT_OP_CALL ||
            (FBT_FMT3_OP(*instr) == FBT_OP_JMPL &&
            FBT_FMT3_RD(*instr) == FBT_REG_O7)) {
                /*
                 * If this is a call (or a jmpl that links into %o7), we can
                 * patch it iff the next instruction uses %o7 as a destination
                 * register.  Because there is an ABI responsibility to
                 * restore %o7 to the value before the call/jmpl, we don't
                 * particularly care how this routine is managing to restore
                 * it (mov, add, ld or divx for all we care).  If it doesn't
                 * seem to be restoring it at all, however, we'll refuse
                 * to patch it.
                 */
                uint32_t delay = *(instr + 1);
                uint32_t op, rd;

                op = FBT_FMT1_OP(delay);
                rd = FBT_FMT3_RD(delay);

                if (op != FBT_OP2 || rd != FBT_REG_O7) {
                        /*
                         * This is odd.  Before we assume that we're looking
                         * at something bizarre (and warn accordingly), we'll
                         * check to see if it's obviously a jump table entry.
                         */
                        if (*instr < (uintptr_t)instr &&
                            *instr >= (uintptr_t)instr - offset)
                                return (0);

                        cmn_err(CE_NOTE, "cannot instrument return of %s at "
                            "%p: leaf jmpl/call delay isn't restoring %%o7",
                            name, (void *)instr);
                        return (0);
                }

                return (1);
        }

        if (offset == sizeof (uint32_t)) {
                /*
                 * If this is the second instruction in the function, we're
                 * going to allow it to be patched if the first instruction
                 * is a patchable return-from-leaf instruction.
                 */
                if (fbt_canpatch_retl(instr - 1, 0, name))
                        return (1);
        }

        if (FBT_FMT3_OP(*instr) != FBT_OP_JMPL)
                return (0);

        if (FBT_FMT3_RD(*instr) != FBT_REG_G0)
                return (0);

        return (1);
}

/*ARGSUSED*/
static uint32_t
fbt_patch_return(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim,
    int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name)
{
        uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
        uint32_t cti = *instr, restore = *(instr + 1), rs1, dest;
        uintptr_t va = tramp->fbtt_va;
        uintptr_t base = tramp->fbtt_next;
        uint32_t locals[FBT_REG_NLOCALS], local;

        if (tramp->fbtt_next + FBT_RETENT_MAXSIZE > tramp->fbtt_limit) {
                /*
                 * There isn't sufficient room for this entry; return failure.
                 */
                return (FBT_ILLTRAP);
        }

        FBT_COUNTER(id, fbt_ret);

        if (FBT_FMT3_OP(*instr) == FBT_OP_RETURN) {
                /*
                 * To handle the case of the return instruction, we'll emit a
                 * restore, followed by the instruction in the slot (which
                 * we'll transplant here), and then another save.  While it
                 * may seem intellectually unsatisfying to emit the additional
                 * restore/save couplet, one can take solace in the fact that
                 * we don't do this if the instruction in the return delay
                 * slot is a nop -- which it is nearly 90% of the time with
                 * gcc.  (And besides, this couplet can't induce unnecessary
                 * spill/fill traps; rewriting the delay instruction to be
                 * in terms of the current window hardly seems worth the
                 * trouble -- let alone the risk.)
                 */
                uint32_t delay = *(instr + 1);
                ASSERT(*instr == FBT_RETURN(FBT_REG_I7, 8));

                cti = FBT_RET;
                restore = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);

                if (delay != FBT_SETHI(0, FBT_REG_G0)) {
                        *tinstr++ = restore;
                        *tinstr++ = delay;
                        *tinstr++ = FBT_SAVEIMM(FBT_REG_O6,
                            -SA(MINFRAME), FBT_REG_O6);
                }
        }

        FBT_REG_INITLOCALS(local, locals);

        /*
         * Mark the locals used in the jmpl.
         */
        if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
                uint32_t rs1 = FBT_FMT3_RS1(cti);
                FBT_REG_MARKLOCAL(locals, rs1);

                if (!FBT_FMT3_ISIMM(cti)) {
                        uint32_t rs2 = FBT_FMT3_RS2(cti);
                        FBT_REG_MARKLOCAL(locals, rs2);
                }
        }

        /*
         * And mark the locals used in the restore.
         */
        rs1 = FBT_FMT3_RS1(restore);
        FBT_REG_MARKLOCAL(locals, rs1);

        if (!FBT_FMT3_ISIMM(restore)) {
                uint32_t rs2 = FBT_FMT3_RS2(restore);
                FBT_REG_MARKLOCAL(locals, rs2);
        }

        if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
                uint32_t rs1 = FBT_FMT3_RS1(cti);

                if (FBT_REG_ISVOLATILE(rs1)) {
                        FBT_REG_ALLOCLOCAL(local, locals);
                        FBT_FMT3_RS1_SET(cti, local);
                        *tinstr++ = FBT_MOV(rs1, local);
                }

                if (!FBT_FMT3_ISIMM(cti)) {
                        uint32_t rs2 = FBT_FMT3_RS2(cti);

                        if (FBT_REG_ISVOLATILE(rs2)) {
                                FBT_REG_ALLOCLOCAL(local, locals);
                                FBT_FMT3_RS2_SET(cti, local);
                                *tinstr++ = FBT_MOV(rs2, local);
                        }
                }
        }

        rs1 = FBT_FMT3_RS1(restore);

        if (FBT_REG_ISVOLATILE(rs1)) {
                FBT_REG_ALLOCLOCAL(local, locals);
                FBT_FMT3_RS1_SET(restore, local);
                *tinstr++ = FBT_MOV(rs1, local);
        }

        if (!FBT_FMT3_ISIMM(restore)) {
                uint32_t rs2 = FBT_FMT3_RS2(restore);

                if (FBT_REG_ISVOLATILE(rs2)) {
                        FBT_REG_ALLOCLOCAL(local, locals);
                        FBT_FMT3_RS2_SET(restore, local);
                        *tinstr++ = FBT_MOV(rs2, local);
                }
        }

        if (id > (uint32_t)FBT_SIMM13_MAX) {
                *tinstr++ = FBT_SETHI(id, FBT_REG_O0);
                *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
        } else {
                *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
        }

        if (offset > (uint32_t)FBT_SIMM13_MAX) {
                *tinstr++ = FBT_SETHI(offset, FBT_REG_O1);
                *tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1);
        } else {
                *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1);
        }

        *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
        tinstr++;

        if (FBT_FMT3_RD(restore) == FBT_REG_O0) {
                /*
                 * If the destination register of the restore is %o0, we
                 * need to perform the implied calculation to derive the
                 * return value.
                 */
                uint32_t add = (restore & ~FBT_FMT3_OP_MASK) | FBT_OP_ADD;
                add &= ~FBT_FMT3_RD_MASK;
                *tinstr++ = add | (FBT_REG_O2 << FBT_FMT3_RD_SHIFT);
        } else {
                *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2);
        }

        /*
         * If the control transfer instruction is %pc-relative (i.e. a
         * call), we need to reset it appropriately.
         */
        if (FBT_FMT1_OP(cti) == FBT_OP_CALL) {
                dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2);
                *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest);
                tinstr++;
        } else {
                *tinstr++ = cti;
        }

        *tinstr++ = restore;
        tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
        tramp->fbtt_next = (uintptr_t)tinstr;

        return (FBT_BAA(instr, va));
}

static uint32_t
fbt_patch_retl(uint32_t *instr, uint32_t *funcbase, uint32_t *funclim,
    int offset, uint32_t id, fbt_trampoline_t *tramp, const char *name)
{
        uint32_t *tinstr = (uint32_t *)tramp->fbtt_next;
        uintptr_t va = tramp->fbtt_va;
        uintptr_t base = tramp->fbtt_next;
        uint32_t cti = *instr, dest;
        int annul = 0;

        FBT_COUNTER(id, fbt_retl);

        if (tramp->fbtt_next + FBT_RETLENT_MAXSIZE > tramp->fbtt_limit) {
                /*
                 * There isn't sufficient room for this entry; return failure.
                 */
                return (FBT_ILLTRAP);
        }

        if (offset == sizeof (uint32_t) &&
            fbt_canpatch_retl(instr - 1, 0, name)) {
                *tinstr++ = *instr;
                annul = 1;
                FBT_COUNTER(id, fbt_retl_twoinstr);
        } else {
                if (FBT_FMT3_OP(cti) == FBT_OP_JMPL &&
                    FBT_FMT3_RD(cti) != FBT_REG_O7 &&
                    FBT_FMT3_RS1(cti) != FBT_REG_O7) {
                        annul = 1;
                        *tinstr++ = *(instr + 1);
                }
        }

        *tinstr++ = FBT_SAVEIMM(FBT_REG_O6, -SA(MINFRAME), FBT_REG_O6);

        if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
                uint32_t rs1, rs2, o2i = FBT_REG_I0 - FBT_REG_O0;

                /*
                 * If we have a jmpl and it's in terms of output registers, we
                 * need to rewrite it to be in terms of the corresponding input
                 * registers.  If it's in terms of the globals, we'll rewrite
                 * it to be in terms of locals.
                 */
                rs1 = FBT_FMT3_RS1(cti);

                if (FBT_REG_ISOUTPUT(rs1))
                        rs1 += o2i;

                if (FBT_REG_ISGLOBAL(rs1)) {
                        *tinstr++ = FBT_MOV(rs1, FBT_REG_L0);
                        rs1 = FBT_REG_L0;
                }

                FBT_FMT3_RS1_SET(cti, rs1);

                if (!FBT_FMT3_ISIMM(cti)) {
                        rs2 = FBT_FMT3_RS2(cti);

                        if (FBT_REG_ISOUTPUT(rs2))
                                rs2 += o2i;

                        if (FBT_REG_ISGLOBAL(rs2)) {
                                *tinstr++ = FBT_MOV(rs2, FBT_REG_L1);
                                rs2 = FBT_REG_L1;
                        }

                        FBT_FMT3_RS2_SET(cti, rs2);
                }

                /*
                 * Now we need to check the rd and source register for the jmpl;
                 * If neither rd nor the source register is %o7, then we might
                 * have a jmp that is actually part of a jump table.  We need
                 * to generate the code to compare it to the base and limit of
                 * the function.
                 */
                if (FBT_FMT3_RD(cti) != FBT_REG_O7 && rs1 != FBT_REG_I7) {
                        uintptr_t base = (uintptr_t)funcbase;
                        uintptr_t limit = (uintptr_t)funclim;

                        FBT_COUNTER(id, fbt_retl_jmptab);

                        if (FBT_FMT3_ISIMM(cti)) {
                                *tinstr++ = FBT_ADDSIMM13(rs1,
                                    FBT_FMT3_SIMM13(cti), FBT_REG_L2);
                        } else {
                                *tinstr++ = FBT_ADD(rs1, rs2, FBT_REG_L2);
                        }

                        *tinstr++ = FBT_SETHI(base, FBT_REG_L3);
                        *tinstr++ = FBT_ORLO(FBT_REG_L3, base, FBT_REG_L3);
                        *tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3);
                        *tinstr++ = FBT_BL(0, 8 * sizeof (uint32_t));
                        *tinstr++ = FBT_SETHI(limit, FBT_REG_L3);
                        *tinstr++ = FBT_ORLO(FBT_REG_L3, limit, FBT_REG_L3);
                        *tinstr++ = FBT_CMP(FBT_REG_L2, FBT_REG_L3);
                        *tinstr++ = FBT_BGE(0, 4 * sizeof (uint32_t));
                        *tinstr++ = FBT_SETHI(0, FBT_REG_G0);
                        *tinstr++ = cti;
                        *tinstr++ = FBT_RESTORE(FBT_REG_G0,
                            FBT_REG_G0, FBT_REG_G0);
                }
        }

        if (id > (uint32_t)FBT_SIMM13_MAX) {
                *tinstr++ = FBT_SETHI(id, FBT_REG_O0);
                *tinstr++ = FBT_ORLO(FBT_REG_O0, id, FBT_REG_O0);
        } else {
                *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, id, FBT_REG_O0);
        }

        if (offset > (uint32_t)FBT_SIMM13_MAX) {
                *tinstr++ = FBT_SETHI(offset, FBT_REG_O1);
                *tinstr++ = FBT_ORLO(FBT_REG_O1, offset, FBT_REG_O1);
        } else {
                *tinstr++ = FBT_ORSIMM13(FBT_REG_G0, offset, FBT_REG_O1);
        }

        *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dtrace_probe);
        tinstr++;
        *tinstr++ = FBT_MOV(FBT_REG_I0, FBT_REG_O2);

        /*
         * If the control transfer instruction is %pc-relative (i.e. a
         * call), we need to reset it appropriately.
         */
        if (FBT_FMT1_OP(cti) == FBT_OP_CALL) {
                FBT_COUNTER(id, fbt_retl_tailcall);
                dest = (uintptr_t)instr + (FBT_FMT1_DISP30(cti) << 2);
                *tinstr = FBT_CALL((uintptr_t)tinstr - base + va, dest);
                tinstr++;
                annul = 1;
        } else {
                if (FBT_FMT3_OP(cti) == FBT_OP_JMPL) {
                        *tinstr++ = cti;

                        if (FBT_FMT3_RD(cti) == FBT_REG_O7) {
                                FBT_COUNTER(id, fbt_retl_tailjmpl);
                                annul = 1;
                        }
                } else {
                        *tinstr++ = FBT_RET;
                }
        }

        *tinstr++ = FBT_RESTORE(FBT_REG_G0, FBT_REG_G0, FBT_REG_G0);

        tramp->fbtt_va += (uintptr_t)tinstr - tramp->fbtt_next;
        tramp->fbtt_next = (uintptr_t)tinstr;

        return (annul ? FBT_BAA(instr, va) : FBT_BA(instr, va));
}

/*ARGSUSED*/
static void
fbt_provide_module(void *arg, struct modctl *ctl)
{
        struct module *mp = ctl->mod_mp;
        char *modname = ctl->mod_modname;
        char *str = mp->strings;
        int nsyms = mp->nsyms;
        Shdr *symhdr = mp->symhdr;
        size_t symsize;
        char *name;
        int i;
        fbt_probe_t *fbt, *retfbt;
        fbt_trampoline_t tramp;
        uintptr_t offset;
        int primary = 0;
        ctf_file_t *fp = NULL;
        int error;
        int estimate = 1;
        uint32_t faketramp[50];
        size_t fbt_size = 0;

        /*
         * Employees of dtrace and their families are ineligible.  Void
         * where prohibited.
         */
        if (strcmp(modname, "dtrace") == 0)
                return;

        if (ctl->mod_requisites != NULL) {
                struct modctl_list *list;

                list = (struct modctl_list *)ctl->mod_requisites;

                for (; list != NULL; list = list->modl_next) {
                        if (strcmp(list->modl_modp->mod_modname, "dtrace") == 0)
                                return;
                }
        }

        /*
         * KMDB is ineligible for instrumentation -- it may execute in
         * any context, including probe context.
         */
        if (strcmp(modname, "kmdbmod") == 0)
                return;

        if (str == NULL || symhdr == NULL || symhdr->sh_addr == 0) {
                /*
                 * If this module doesn't (yet) have its string or symbol
                 * table allocated, clear out.
                 */
                return;
        }

        symsize = symhdr->sh_entsize;

        if (mp->fbt_nentries) {
                /*
                 * This module has some FBT entries allocated; we're afraid
                 * to screw with it.
                 */
                return;
        }

        if (mp->fbt_tab != NULL)
                estimate = 0;

        /*
         * This is a hack for unix/genunix/krtld.
         */
        primary = vmem_contains(heap_arena, (void *)ctl,
            sizeof (struct modctl)) == 0;
        kobj_textwin_alloc(mp);

        /*
         * Open the CTF data for the module.  We'll use this to determine the
         * functions that can be instrumented.  Note that this call can fail,
         * in which case we'll use heuristics to determine the functions that
         * can be instrumented.  (But in particular, leaf functions will not be
         * instrumented.)
         */
        fp = ctf_modopen(mp, &error);

forreal:
        if (!estimate) {
                tramp.fbtt_next =
                    (uintptr_t)fbt_trampoline_map((uintptr_t)mp->fbt_tab,
                    mp->fbt_size);
                tramp.fbtt_limit = tramp.fbtt_next + mp->fbt_size;
                tramp.fbtt_va = (uintptr_t)mp->fbt_tab;
        }

        for (i = 1; i < nsyms; i++) {
                ctf_funcinfo_t f;
                uint32_t *instr, *base, *limit;
                Sym *sym = (Sym *)(symhdr->sh_addr + i * symsize);
                int have_ctf = 0, is_leaf = 0, nargs, cti = 0;
                int (*canpatch)(uint32_t *, int, const char *);
                uint32_t (*patch)(uint32_t *, uint32_t *, uint32_t *, int,
                    uint32_t, fbt_trampoline_t *, const char *);

                if (ELF_ST_TYPE(sym->st_info) != STT_FUNC)
                        continue;

                /*
                 * Weak symbols are not candidates.  This could be made to
                 * work (where weak functions and their underlying function
                 * appear as two disjoint probes), but it's not simple.
                 */
                if (ELF_ST_BIND(sym->st_info) == STB_WEAK)
                        continue;

                name = str + sym->st_name;

                if (strstr(name, "dtrace_") == name &&
                    strstr(name, "dtrace_safe_") != name) {
                        /*
                         * Anything beginning with "dtrace_" may be called
                         * from probe context unless it explitly indicates
                         * that it won't be called from probe context by
                         * using the prefix "dtrace_safe_".
                         */
                        continue;
                }

                if (strstr(name, "kdi_") == name ||
                    strstr(name, "_kdi_") != NULL) {
                        /*
                         * Any function name beginning with "kdi_" or
                         * containing the string "_kdi_" is a part of the
                         * kernel debugger interface and may be called in
                         * arbitrary context -- including probe context.
                         */
                        continue;
                }

                if (strstr(name, "__relocatable") != NULL) {
                        /*
                         * Anything with the string "__relocatable" anywhere
                         * in the function name is considered to be a function
                         * that may be manually relocated before execution.
                         * Because FBT uses a PC-relative technique for
                         * instrumentation, these functions cannot safely
                         * be instrumented by us.
                         */
                        continue;
                }

                if (strstr(name, "ip_ocsum") == name) {
                        /*
                         * The ip_ocsum_* family of routines are all ABI
                         * violators.  (They expect incoming arguments in the
                         * globals!)  Break the ABI?  No soup for you!
                         */
                        continue;
                }

                /*
                 * We want to scan the function for one (and only one) save.
                 * Any more indicates that something fancy is going on.
                 */
                base = (uint32_t *)sym->st_value;
                limit = (uint32_t *)(sym->st_value + sym->st_size);

                /*
                 * We don't want to interpose on the module stubs.
                 */
                if (base >= (uint32_t *)stubs_base &&
                    base <= (uint32_t *)stubs_end)
                        continue;

                /*
                 * We can't safely trace a zero-length function...
                 */
                if (base == limit)
                        continue;

                /*
                 * Due to 4524008, _init and _fini may have a bloated st_size.
                 * While this bug was fixed quite some time ago, old drivers
                 * may be lurking.  We need to develop a better solution to
                 * this problem, such that correct _init and _fini functions
                 * (the vast majority) may be correctly traced.  One solution
                 * may be to scan through the entire symbol table to see if
                 * any symbol overlaps with _init.  If none does, set a bit in
                 * the module structure that this module has correct _init and
                 * _fini sizes.  This will cause some pain the first time a
                 * module is scanned, but at least it would be O(N) instead of
                 * O(N log N)...
                 */
                if (strcmp(name, "_init") == 0)
                        continue;

                if (strcmp(name, "_fini") == 0)
                        continue;

                instr = base;

                /*
                 * While we try hard to only trace safe functions (that is,
                 * functions at TL=0), one unsafe function manages to otherwise
                 * appear safe:  prom_trap().  We could discover prom_trap()
                 * if we added an additional rule:  in order to trace a
                 * function, we must either (a) discover a restore or (b)
                 * determine that the function does not have any unlinked
                 * control transfers to another function (i.e., the function
                 * never returns).  Unfortunately, as of this writing, one
                 * legitimate function (resume_from_zombie()) transfers
                 * control to a different function (_resume_from_idle())
                 * without executing a restore.  Barring a rule to figure out
                 * that resume_from_zombie() is safe while prom_trap() is not,
                 * we resort to hard-coding prom_trap() here.
                 */
                if (strcmp(name, "prom_trap") == 0)
                        continue;

                if (fp != NULL && ctf_func_info(fp, i, &f) != CTF_ERR) {
                        nargs = f.ctc_argc;
                        have_ctf = 1;
                } else {
                        nargs = 32;
                }

                /*
                 * If the first instruction of the function is a branch and
                 * it's not a branch-always-not-annulled, we're going to refuse
                 * to patch it.
                 */
                if ((*instr & FBT_OP_MASK) == FBT_OP0 &&
                    (*instr & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_SETHI &&
                    (*instr & FBT_FMT2_OP2_MASK) != FBT_FMT2_OP2_BPR) {
                        if (!FBT_IS_BA(*instr) && !FBT_IS_BAPCC(*instr)) {
                                if (have_ctf) {
                                        cmn_err(CE_NOTE, "cannot instrument %s:"
                                            " begins with non-ba, "
                                            "non-br CTI", name);
                                }
                                continue;
                        }
                }

                while (!FBT_IS_SAVE(*instr)) {
                        /*
                         * Before we assume that this is a leaf routine, check
                         * forward in the basic block for a save.
                         */
                        int op = *instr & FBT_OP_MASK;
                        int op2 = *instr & FBT_FMT2_OP2_MASK;

                        if (op == FBT_OP0 && op2 != FBT_FMT2_OP2_SETHI) {
                                /*
                                 * This is a CTI.  If we see a subsequent
                                 * save, we will refuse to process this
                                 * routine unless both of the following are
                                 * true:
                                 *
                                 *  (a) The branch is not annulled
                                 *
                                 *  (b) The subsequent save is in the delay
                                 *      slot of the branch
                                 */
                                if ((*instr & FBT_ANNUL) ||
                                    !FBT_IS_SAVE(*(instr + 1))) {
                                        cti = 1;
                                } else {
                                        instr++;
                                        break;
                                }
                        }

                        if (op == FBT_OP1)
                                cti = 1;

                        if (++instr == limit)
                                break;
                }

                if (instr < limit && cti) {
                        /*
                         * If we found a CTI before the save, we need to not
                         * do anything.  But if we have CTF information, this
                         * is weird enough that it merits a message.
                         */
                        if (!have_ctf)
                                continue;

                        cmn_err(CE_NOTE, "cannot instrument %s: "
                            "save not in first basic block", name);
                        continue;
                }

                if (instr == limit) {
                        if (!have_ctf)
                                continue;
                        is_leaf = 1;

                        if (!estimate)
                                fbt_leaf_functions++;

                        canpatch = fbt_canpatch_retl;
                        patch = fbt_patch_retl;
                } else {
                        canpatch = fbt_canpatch_return;
                        patch = fbt_patch_return;
                }

                if (!have_ctf && !is_leaf) {
                        /*
                         * Before we assume that this isn't something tricky,
                         * look for other saves.  If we find them, there are
                         * multiple entry points here (or something), and we'll
                         * leave it alone.
                         */
                        while (++instr < limit) {
                                if (FBT_IS_SAVE(*instr))
                                        break;
                        }

                        if (instr != limit)
                                continue;
                }

                instr = base;

                if (FBT_IS_CTI(*instr)) {
                        /*
                         * If we have a CTI, we want to be sure that we don't
                         * have a CTI or a PC-relative instruction in the
                         * delay slot -- we want to be able to thunk the
                         * instruction into the trampoline without worrying
                         * about either DCTIs or relocations.  It would be
                         * very odd for the compiler to generate this kind of
                         * code, so we warn about it if we have CTF
                         * information.
                         */
                        if (FBT_IS_CTI(*(instr + 1))) {
                                if (!have_ctf)
                                        continue;

                                cmn_err(CE_NOTE, "cannot instrument %s: "
                                    "CTI in delay slot of first instruction",
                                    name);
                                continue;
                        }

                        if (FBT_IS_PCRELATIVE(*(instr + 1))) {
                                if (!have_ctf)
                                        continue;

                                cmn_err(CE_NOTE, "cannot instrument %s: "
                                    "PC-relative instruction in delay slot of"
                                    " first instruction", name);
                                continue;
                        }
                }

                if (estimate) {
                        tramp.fbtt_next = (uintptr_t)faketramp;
                        tramp.fbtt_limit = tramp.fbtt_next + sizeof (faketramp);
                        (void) fbt_patch_entry(instr, FBT_ESTIMATE_ID,
                            &tramp, nargs);
                        fbt_size += tramp.fbtt_next - (uintptr_t)faketramp;
                } else {
                        fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
                        fbt->fbtp_name = name;
                        fbt->fbtp_ctl = ctl;
                        fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
                            name, FBT_PROBENAME_ENTRY, 1, fbt);
                        fbt->fbtp_patchval = FBT_BAA(instr, tramp.fbtt_va);

                        if (!fbt_patch_entry(instr, fbt->fbtp_id,
                            &tramp, nargs)) {
                                cmn_err(CE_WARN, "unexpectedly short FBT table "
                                    "in module %s (sym %d of %d)", modname,
                                    i, nsyms);
                                break;
                        }

                        fbt->fbtp_patchpoint =
                            (uint32_t *)((uintptr_t)mp->textwin +
                            ((uintptr_t)instr - (uintptr_t)mp->text));
                        fbt->fbtp_savedval = *instr;

                        fbt->fbtp_loadcnt = ctl->mod_loadcnt;
                        fbt->fbtp_primary = primary;
                        fbt->fbtp_symndx = i;
                        mp->fbt_nentries++;
                }

                retfbt = NULL;
again:
                if (++instr == limit)
                        continue;

                offset = (uintptr_t)instr - (uintptr_t)base;

                if (!(*canpatch)(instr, offset, name))
                        goto again;

                if (estimate) {
                        tramp.fbtt_next = (uintptr_t)faketramp;
                        tramp.fbtt_limit = tramp.fbtt_next + sizeof (faketramp);
                        (void) (*patch)(instr, base, limit,
                            offset, FBT_ESTIMATE_ID, &tramp, name);
                        fbt_size += tramp.fbtt_next - (uintptr_t)faketramp;

                        goto again;
                }

                fbt = kmem_zalloc(sizeof (fbt_probe_t), KM_SLEEP);
                fbt->fbtp_name = name;
                fbt->fbtp_ctl = ctl;

                if (retfbt == NULL) {
                        fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
                            name, FBT_PROBENAME_RETURN, 1, fbt);
                } else {
                        retfbt->fbtp_next = fbt;
                        fbt->fbtp_id = retfbt->fbtp_id;
                }

                fbt->fbtp_return = 1;
                retfbt = fbt;

                if ((fbt->fbtp_patchval = (*patch)(instr, base, limit, offset,
                    fbt->fbtp_id, &tramp, name)) == FBT_ILLTRAP) {
                        cmn_err(CE_WARN, "unexpectedly short FBT table "
                            "in module %s (sym %d of %d)", modname, i, nsyms);
                        break;
                }

                fbt->fbtp_patchpoint = (uint32_t *)((uintptr_t)mp->textwin +
                    ((uintptr_t)instr - (uintptr_t)mp->text));
                fbt->fbtp_savedval = *instr;
                fbt->fbtp_loadcnt = ctl->mod_loadcnt;
                fbt->fbtp_primary = primary;
                fbt->fbtp_symndx = i;
                mp->fbt_nentries++;

                goto again;
        }

        if (estimate) {
                /*
                 * Slosh on another entry's worth...
                 */
                fbt_size += FBT_ENT_MAXSIZE;
                mp->fbt_size = fbt_size;
                mp->fbt_tab = kobj_texthole_alloc(mp->text, fbt_size);

                if (mp->fbt_tab == NULL) {
                        cmn_err(CE_WARN, "couldn't allocate FBT table "
                            "for module %s", modname);
                } else {
                        estimate = 0;
                        goto forreal;
                }
        } else {
                fbt_trampoline_unmap();
        }

error:
        if (fp != NULL)
                ctf_close(fp);
}

/*ARGSUSED*/
static void
fbt_destroy(void *arg, dtrace_id_t id, void *parg)
{
        fbt_probe_t *fbt = parg, *next;
        struct modctl *ctl = fbt->fbtp_ctl;

        do {
                if (ctl != NULL && ctl->mod_loadcnt == fbt->fbtp_loadcnt) {
                        if ((ctl->mod_loadcnt == fbt->fbtp_loadcnt &&
                            ctl->mod_loaded) || fbt->fbtp_primary) {
                                ((struct module *)
                                    (ctl->mod_mp))->fbt_nentries--;
                        }
                }

                next = fbt->fbtp_next;
                kmem_free(fbt, sizeof (fbt_probe_t));
                fbt = next;
        } while (fbt != NULL);
}

/*ARGSUSED*/
static int
fbt_enable(void *arg, dtrace_id_t id, void *parg)
{
        fbt_probe_t *fbt = parg, *f;
        struct modctl *ctl = fbt->fbtp_ctl;

        ctl->mod_nenabled++;

        for (f = fbt; f != NULL; f = f->fbtp_next) {
                if (f->fbtp_patchpoint == NULL) {
                        /*
                         * Due to a shortened FBT table, this entry was never
                         * completed; refuse to enable it.
                         */
                        if (fbt_verbose) {
                                cmn_err(CE_NOTE, "fbt is failing for probe %s "
                                    "(short FBT table in %s)",
                                    fbt->fbtp_name, ctl->mod_modname);
                        }

                        return (0);
                }
        }

        /*
         * If this module has disappeared since we discovered its probes,
         * refuse to enable it.
         */
        if (!fbt->fbtp_primary && !ctl->mod_loaded) {
                if (fbt_verbose) {
                        cmn_err(CE_NOTE, "fbt is failing for probe %s "
                            "(module %s unloaded)",
                            fbt->fbtp_name, ctl->mod_modname);
                }

                return (0);
        }

        /*
         * Now check that our modctl has the expected load count.  If it
         * doesn't, this module must have been unloaded and reloaded -- and
         * we're not going to touch it.
         */
        if (ctl->mod_loadcnt != fbt->fbtp_loadcnt) {
                if (fbt_verbose) {
                        cmn_err(CE_NOTE, "fbt is failing for probe %s "
                            "(module %s reloaded)",
                            fbt->fbtp_name, ctl->mod_modname);
                }

                return (0);
        }

        for (; fbt != NULL; fbt = fbt->fbtp_next)
                *fbt->fbtp_patchpoint = fbt->fbtp_patchval;

        return (0);
}

/*ARGSUSED*/
static void
fbt_disable(void *arg, dtrace_id_t id, void *parg)
{
        fbt_probe_t *fbt = parg, *f;
        struct modctl *ctl = fbt->fbtp_ctl;

        ASSERT(ctl->mod_nenabled > 0);
        ctl->mod_nenabled--;

        for (f = fbt; f != NULL; f = f->fbtp_next) {
                if (f->fbtp_patchpoint == NULL)
                        return;
        }

        if ((!fbt->fbtp_primary && !ctl->mod_loaded) ||
            (ctl->mod_loadcnt != fbt->fbtp_loadcnt))
                return;

        for (; fbt != NULL; fbt = fbt->fbtp_next)
                *fbt->fbtp_patchpoint = fbt->fbtp_savedval;
}

/*ARGSUSED*/
static void
fbt_suspend(void *arg, dtrace_id_t id, void *parg)
{
        fbt_probe_t *fbt = parg;
        struct modctl *ctl = fbt->fbtp_ctl;

        if (!fbt->fbtp_primary && !ctl->mod_loaded)
                return;

        if (ctl->mod_loadcnt != fbt->fbtp_loadcnt)
                return;

        ASSERT(ctl->mod_nenabled > 0);

        for (; fbt != NULL; fbt = fbt->fbtp_next)
                *fbt->fbtp_patchpoint = fbt->fbtp_savedval;
}

/*ARGSUSED*/
static void
fbt_resume(void *arg, dtrace_id_t id, void *parg)
{
        fbt_probe_t *fbt = parg;
        struct modctl *ctl = fbt->fbtp_ctl;

        if (!fbt->fbtp_primary && !ctl->mod_loaded)
                return;

        if (ctl->mod_loadcnt != fbt->fbtp_loadcnt)
                return;

        ASSERT(ctl->mod_nenabled > 0);

        for (; fbt != NULL; fbt = fbt->fbtp_next)
                *fbt->fbtp_patchpoint = fbt->fbtp_patchval;
}

/*ARGSUSED*/
static void
fbt_getargdesc(void *arg, dtrace_id_t id, void *parg, dtrace_argdesc_t *desc)
{
        fbt_probe_t *fbt = parg;
        struct modctl *ctl = fbt->fbtp_ctl;
        struct module *mp = ctl->mod_mp;
        ctf_file_t *fp = NULL, *pfp;
        ctf_funcinfo_t f;
        int error;
        ctf_id_t argv[32], type;
        int argc = sizeof (argv) / sizeof (ctf_id_t);
        const char *parent;

        if (!ctl->mod_loaded || (ctl->mod_loadcnt != fbt->fbtp_loadcnt))
                goto err;

        if (fbt->fbtp_return && desc->dtargd_ndx == 0) {
                (void) strcpy(desc->dtargd_native, "int");
                return;
        }

        if ((fp = ctf_modopen(mp, &error)) == NULL) {
                /*
                 * We have no CTF information for this module -- and therefore
                 * no args[] information.
                 */
                goto err;
        }

        /*
         * If we have a parent container, we must manually import it.
         */
        if ((parent = ctf_parent_name(fp)) != NULL) {
                struct modctl *mp = &modules;
                struct modctl *mod = NULL;

                /*
                 * We must iterate over all modules to find the module that
                 * is our parent.
                 */
                do {
                        if (strcmp(mp->mod_modname, parent) == 0) {
                                mod = mp;
                                break;
                        }
                } while ((mp = mp->mod_next) != &modules);

                if (mod == NULL)
                        goto err;

                if ((pfp = ctf_modopen(mod->mod_mp, &error)) == NULL)
                        goto err;

                if (ctf_import(fp, pfp) != 0) {
                        ctf_close(pfp);
                        goto err;
                }

                ctf_close(pfp);
        }

        if (ctf_func_info(fp, fbt->fbtp_symndx, &f) == CTF_ERR)
                goto err;

        if (fbt->fbtp_return) {
                if (desc->dtargd_ndx > 1)
                        goto err;

                ASSERT(desc->dtargd_ndx == 1);
                type = f.ctc_return;
        } else {
                if (desc->dtargd_ndx + 1 > f.ctc_argc)
                        goto err;

                if (ctf_func_args(fp, fbt->fbtp_symndx, argc, argv) == CTF_ERR)
                        goto err;

                type = argv[desc->dtargd_ndx];
        }

        if (ctf_type_name(fp, type, desc->dtargd_native,
            DTRACE_ARGTYPELEN) != NULL) {
                ctf_close(fp);
                return;
        }
err:
        if (fp != NULL)
                ctf_close(fp);

        desc->dtargd_ndx = DTRACE_ARGNONE;
}

static dtrace_pattr_t fbt_attr = {
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_UNKNOWN },
{ DTRACE_STABILITY_EVOLVING, DTRACE_STABILITY_EVOLVING, DTRACE_CLASS_ISA },
{ DTRACE_STABILITY_PRIVATE, DTRACE_STABILITY_PRIVATE, DTRACE_CLASS_ISA },
};

static dtrace_pops_t fbt_pops = {
        NULL,
        fbt_provide_module,
        fbt_enable,
        fbt_disable,
        fbt_suspend,
        fbt_resume,
        fbt_getargdesc,
        NULL,
        NULL,
        fbt_destroy
};

static int
fbt_attach(dev_info_t *devi, ddi_attach_cmd_t cmd)
{
        switch (cmd) {
        case DDI_ATTACH:
                break;
        case DDI_RESUME:
                return (DDI_SUCCESS);
        default:
                return (DDI_FAILURE);
        }

        if (ddi_create_minor_node(devi, "fbt", S_IFCHR, 0,
            DDI_PSEUDO, 0) == DDI_FAILURE ||
            dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_KERNEL, NULL,
            &fbt_pops, NULL, &fbt_id) != 0) {
                ddi_remove_minor_node(devi, NULL);
                return (DDI_FAILURE);
        }

        ddi_report_dev(devi);
        fbt_devi = devi;
        return (DDI_SUCCESS);
}

static int
fbt_detach(dev_info_t *devi, ddi_detach_cmd_t cmd)
{
        switch (cmd) {
        case DDI_DETACH:
                break;
        case DDI_SUSPEND:
                return (DDI_SUCCESS);
        default:
                return (DDI_FAILURE);
        }

        if (dtrace_unregister(fbt_id) != 0)
                return (DDI_FAILURE);

        ddi_remove_minor_node(devi, NULL);
        return (DDI_SUCCESS);
}

/*ARGSUSED*/
static int
fbt_info(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
{
        int error;

        switch (infocmd) {
        case DDI_INFO_DEVT2DEVINFO:
                *result = (void *)fbt_devi;
                error = DDI_SUCCESS;
                break;
        case DDI_INFO_DEVT2INSTANCE:
                *result = (void *)0;
                error = DDI_SUCCESS;
                break;
        default:
                error = DDI_FAILURE;
        }
        return (error);
}

/*ARGSUSED*/
static int
fbt_open(dev_t *devp, int flag, int otyp, cred_t *cred_p)
{
        return (0);
}

static struct cb_ops fbt_cb_ops = {
        fbt_open,               /* open */
        nodev,                  /* close */
        nulldev,                /* strategy */
        nulldev,                /* print */
        nodev,                  /* dump */
        nodev,                  /* read */
        nodev,                  /* write */
        nodev,                  /* ioctl */
        nodev,                  /* devmap */
        nodev,                  /* mmap */
        nodev,                  /* segmap */
        nochpoll,               /* poll */
        ddi_prop_op,            /* cb_prop_op */
        0,                      /* streamtab  */
        D_NEW | D_MP            /* Driver compatibility flag */
};

static struct dev_ops fbt_ops = {
        DEVO_REV,               /* devo_rev */
        0,                      /* refcnt */
        fbt_info,               /* get_dev_info */
        nulldev,                /* identify */
        nulldev,                /* probe */
        fbt_attach,             /* attach */
        fbt_detach,             /* detach */
        nodev,                  /* reset */
        &fbt_cb_ops,            /* driver operations */
        NULL,                   /* bus operations */
        nodev,                  /* dev power */
        ddi_quiesce_not_needed,         /* quiesce */
};

/*
 * Module linkage information for the kernel.
 */
static struct modldrv modldrv = {
        &mod_driverops,         /* module type (this is a pseudo driver) */
        "Function Boundary Tracing",    /* name of module */
        &fbt_ops,               /* driver ops */
};

static struct modlinkage modlinkage = {
        MODREV_1,
        (void *)&modldrv,
        NULL
};

int
_init(void)
{
        return (mod_install(&modlinkage));
}

int
_info(struct modinfo *modinfop)
{
        return (mod_info(&modlinkage, modinfop));
}

int
_fini(void)
{
        return (mod_remove(&modlinkage));
}