sys/dev/scsipi/scsipi_base.c

root/sys/dev/scsipi/scsipi_base.c
/*      $NetBSD: scsipi_base.c,v 1.193 2024/10/29 15:50:07 nat Exp $    */

/*-
 * Copyright (c) 1998, 1999, 2000, 2002, 2003, 2004 The NetBSD Foundation, Inc.
 * All rights reserved.
 *
 * This code is derived from software contributed to The NetBSD Foundation
 * by Charles M. Hannum; by Jason R. Thorpe of the Numerical Aerospace
 * Simulation Facility, NASA Ames Research Center.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 * POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/cdefs.h>
__KERNEL_RCSID(0, "$NetBSD: scsipi_base.c,v 1.193 2024/10/29 15:50:07 nat Exp $");

#ifdef _KERNEL_OPT
#include "opt_scsi.h"
#endif

#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/buf.h>
#include <sys/uio.h>
#include <sys/malloc.h>
#include <sys/pool.h>
#include <sys/errno.h>
#include <sys/device.h>
#include <sys/proc.h>
#include <sys/kthread.h>
#include <sys/hash.h>
#include <sys/atomic.h>

#include <dev/scsipi/scsi_sdt.h>
#include <dev/scsipi/scsi_spc.h>
#include <dev/scsipi/scsipi_all.h>
#include <dev/scsipi/scsipi_disk.h>
#include <dev/scsipi/scsipiconf.h>
#include <dev/scsipi/scsipi_base.h>

#include <dev/scsipi/scsi_all.h>
#include <dev/scsipi/scsi_message.h>

#include <machine/param.h>

SDT_PROVIDER_DEFINE(scsi);

SDT_PROBE_DEFINE3(scsi, base, tag, get,
    "struct scsipi_xfer *"/*xs*/, "uint8_t"/*tag*/, "uint8_t"/*type*/);
SDT_PROBE_DEFINE3(scsi, base, tag, put,
    "struct scsipi_xfer *"/*xs*/, "uint8_t"/*tag*/, "uint8_t"/*type*/);

SDT_PROBE_DEFINE3(scsi, base, adapter, request__start,
    "struct scsipi_channel *"/*chan*/,
    "scsipi_adapter_req_t"/*req*/,
    "void *"/*arg*/);
SDT_PROBE_DEFINE3(scsi, base, adapter, request__done,
    "struct scsipi_channel *"/*chan*/,
    "scsipi_adapter_req_t"/*req*/,
    "void *"/*arg*/);

SDT_PROBE_DEFINE1(scsi, base, queue, batch__start,
    "struct scsipi_channel *"/*chan*/);
SDT_PROBE_DEFINE2(scsi, base, queue, run,
    "struct scsipi_channel *"/*chan*/,
    "struct scsipi_xfer *"/*xs*/);
SDT_PROBE_DEFINE1(scsi, base, queue, batch__done,
    "struct scsipi_channel *"/*chan*/);

SDT_PROBE_DEFINE1(scsi, base, xfer, execute,  "struct scsipi_xfer *"/*xs*/);
SDT_PROBE_DEFINE1(scsi, base, xfer, enqueue,  "struct scsipi_xfer *"/*xs*/);
SDT_PROBE_DEFINE1(scsi, base, xfer, done,  "struct scsipi_xfer *"/*xs*/);
SDT_PROBE_DEFINE1(scsi, base, xfer, redone,  "struct scsipi_xfer *"/*xs*/);
SDT_PROBE_DEFINE1(scsi, base, xfer, complete,  "struct scsipi_xfer *"/*xs*/);
SDT_PROBE_DEFINE1(scsi, base, xfer, restart,  "struct scsipi_xfer *"/*xs*/);
SDT_PROBE_DEFINE1(scsi, base, xfer, free,  "struct scsipi_xfer *"/*xs*/);

static int      scsipi_complete(struct scsipi_xfer *);
static void     scsipi_request_sense(struct scsipi_xfer *);
static int      scsipi_enqueue(struct scsipi_xfer *);
static void     scsipi_run_queue(struct scsipi_channel *chan);

static void     scsipi_completion_thread(void *);

static void     scsipi_get_tag(struct scsipi_xfer *);
static void     scsipi_put_tag(struct scsipi_xfer *);

static int      scsipi_get_resource(struct scsipi_channel *);
static void     scsipi_put_resource(struct scsipi_channel *);

static void     scsipi_async_event_max_openings(struct scsipi_channel *,
                    struct scsipi_max_openings *);
static void     scsipi_async_event_channel_reset(struct scsipi_channel *);

static void     scsipi_channel_freeze_locked(struct scsipi_channel *, int);

static void     scsipi_adapter_lock(struct scsipi_adapter *adapt);
static void     scsipi_adapter_unlock(struct scsipi_adapter *adapt);

static void     scsipi_update_timeouts(struct scsipi_xfer *xs);

static struct pool scsipi_xfer_pool;

int scsipi_xs_count = 0;

/*
 * scsipi_init:
 *
 *      Called when a scsibus or atapibus is attached to the system
 *      to initialize shared data structures.
 */
void
scsipi_init(void)
{
        static int scsipi_init_done;

        if (scsipi_init_done)
                return;
        scsipi_init_done = 1;

        /* Initialize the scsipi_xfer pool. */
        pool_init(&scsipi_xfer_pool, sizeof(struct scsipi_xfer), 0,
            0, 0, "scxspl", NULL, IPL_BIO);
        pool_prime(&scsipi_xfer_pool, 1);

        scsipi_ioctl_init();
}

/*
 * scsipi_channel_init:
 *
 *      Initialize a scsipi_channel when it is attached.
 */
int
scsipi_channel_init(struct scsipi_channel *chan)
{
        struct scsipi_adapter *adapt = chan->chan_adapter;
        int i;

        /* Initialize shared data. */
        scsipi_init();

        /* Initialize the queues. */
        TAILQ_INIT(&chan->chan_queue);
        TAILQ_INIT(&chan->chan_complete);

        for (i = 0; i < SCSIPI_CHAN_PERIPH_BUCKETS; i++)
                LIST_INIT(&chan->chan_periphtab[i]);

        /*
         * Create the asynchronous completion thread.
         */
        if (kthread_create(PRI_NONE, 0, NULL, scsipi_completion_thread, chan,
            &chan->chan_thread, "%s", chan->chan_name)) {
                aprint_error_dev(adapt->adapt_dev, "unable to create completion thread for "
                    "channel %d\n", chan->chan_channel);
                panic("scsipi_channel_init");
        }

        return 0;
}

/*
 * scsipi_channel_shutdown:
 *
 *      Shutdown a scsipi_channel.
 */
void
scsipi_channel_shutdown(struct scsipi_channel *chan)
{

        mutex_enter(chan_mtx(chan));
        /*
         * Shut down the completion thread.
         */
        chan->chan_tflags |= SCSIPI_CHANT_SHUTDOWN;
        cv_broadcast(chan_cv_complete(chan));

        /*
         * Now wait for the thread to exit.
         */
        while (chan->chan_thread != NULL)
                cv_wait(chan_cv_thread(chan), chan_mtx(chan));
        mutex_exit(chan_mtx(chan));
}

static uint32_t
scsipi_chan_periph_hash(uint64_t t, uint64_t l)
{
        uint32_t hash;

        hash = hash32_buf(&t, sizeof(t), HASH32_BUF_INIT);
        hash = hash32_buf(&l, sizeof(l), hash);

        return hash & SCSIPI_CHAN_PERIPH_HASHMASK;
}

/*
 * scsipi_insert_periph:
 *
 *      Insert a periph into the channel.
 */
void
scsipi_insert_periph(struct scsipi_channel *chan, struct scsipi_periph *periph)
{
        uint32_t hash;

        hash = scsipi_chan_periph_hash(periph->periph_target,
            periph->periph_lun);

        mutex_enter(chan_mtx(chan));
        LIST_INSERT_HEAD(&chan->chan_periphtab[hash], periph, periph_hash);
        mutex_exit(chan_mtx(chan));
}

/*
 * scsipi_remove_periph:
 *
 *      Remove a periph from the channel.
 */
void
scsipi_remove_periph(struct scsipi_channel *chan,
    struct scsipi_periph *periph)
{

        LIST_REMOVE(periph, periph_hash);
}

/*
 * scsipi_lookup_periph:
 *
 *      Lookup a periph on the specified channel.
 */
static struct scsipi_periph *
scsipi_lookup_periph_internal(struct scsipi_channel *chan, int target, int lun, bool lock)
{
        struct scsipi_periph *periph;
        uint32_t hash;

        if (target >= chan->chan_ntargets ||
            lun >= chan->chan_nluns)
                return NULL;

        hash = scsipi_chan_periph_hash(target, lun);

        if (lock)
                mutex_enter(chan_mtx(chan));
        LIST_FOREACH(periph, &chan->chan_periphtab[hash], periph_hash) {
                if (periph->periph_target == target &&
                    periph->periph_lun == lun)
                        break;
        }
        if (lock)
                mutex_exit(chan_mtx(chan));

        return periph;
}

struct scsipi_periph *
scsipi_lookup_periph_locked(struct scsipi_channel *chan, int target, int lun)
{
        return scsipi_lookup_periph_internal(chan, target, lun, false);
}

struct scsipi_periph *
scsipi_lookup_periph(struct scsipi_channel *chan, int target, int lun)
{
        return scsipi_lookup_periph_internal(chan, target, lun, true);
}

/*
 * scsipi_get_resource:
 *
 *      Allocate a single xfer `resource' from the channel.
 *
 *      NOTE: Must be called with channel lock held
 */
static int
scsipi_get_resource(struct scsipi_channel *chan)
{
        struct scsipi_adapter *adapt = chan->chan_adapter;

        if (chan->chan_flags & SCSIPI_CHAN_OPENINGS) {
                if (chan->chan_openings > 0) {
                        chan->chan_openings--;
                        return 1;
                }
                return 0;
        }

        if (adapt->adapt_openings > 0) {
                adapt->adapt_openings--;
                return 1;
        }
        return 0;
}

/*
 * scsipi_grow_resources:
 *
 *      Attempt to grow resources for a channel.  If this succeeds,
 *      we allocate one for our caller.
 *
 *      NOTE: Must be called with channel lock held
 */
static inline int
scsipi_grow_resources(struct scsipi_channel *chan)
{

        if (chan->chan_flags & SCSIPI_CHAN_CANGROW) {
                if ((chan->chan_flags & SCSIPI_CHAN_TACTIVE) == 0) {
                        mutex_exit(chan_mtx(chan));
                        scsipi_adapter_request(chan,
                            ADAPTER_REQ_GROW_RESOURCES, NULL);
                        mutex_enter(chan_mtx(chan));
                        return scsipi_get_resource(chan);
                }
                /*
                 * ask the channel thread to do it. It'll have to thaw the
                 * queue
                 */
                scsipi_channel_freeze_locked(chan, 1);
                chan->chan_tflags |= SCSIPI_CHANT_GROWRES;
                cv_broadcast(chan_cv_complete(chan));
                return 0;
        }

        return 0;
}

/*
 * scsipi_put_resource:
 *
 *      Free a single xfer `resource' to the channel.
 *
 *      NOTE: Must be called with channel lock held
 */
static void
scsipi_put_resource(struct scsipi_channel *chan)
{
        struct scsipi_adapter *adapt = chan->chan_adapter;

        if (chan->chan_flags & SCSIPI_CHAN_OPENINGS)
                chan->chan_openings++;
        else
                adapt->adapt_openings++;
}

/*
 * scsipi_get_tag:
 *
 *      Get a tag ID for the specified xfer.
 *
 *      NOTE: Must be called with channel lock held
 */
static void
scsipi_get_tag(struct scsipi_xfer *xs)
{
        struct scsipi_periph *periph = xs->xs_periph;
        int bit, tag;
        u_int word;

        KASSERT(mutex_owned(chan_mtx(periph->periph_channel)));

        bit = 0;        /* XXX gcc */
        for (word = 0; word < PERIPH_NTAGWORDS; word++) {
                bit = ffs(periph->periph_freetags[word]);
                if (bit != 0)
                        break;
        }
#ifdef DIAGNOSTIC
        if (word == PERIPH_NTAGWORDS) {
                scsipi_printaddr(periph);
                printf("no free tags\n");
                panic("scsipi_get_tag");
        }
#endif

        bit -= 1;
        periph->periph_freetags[word] &= ~(1U << bit);
        tag = (word << 5) | bit;

        /* XXX Should eventually disallow this completely. */
        if (tag >= periph->periph_openings) {
                scsipi_printaddr(periph);
                printf("WARNING: tag %d greater than available openings %d\n",
                    tag, periph->periph_openings);
        }

        xs->xs_tag_id = tag;
        SDT_PROBE3(scsi, base, tag, get,
            xs, xs->xs_tag_id, xs->xs_tag_type);
}

/*
 * scsipi_put_tag:
 *
 *      Put the tag ID for the specified xfer back into the pool.
 *
 *      NOTE: Must be called with channel lock held
 */
static void
scsipi_put_tag(struct scsipi_xfer *xs)
{
        struct scsipi_periph *periph = xs->xs_periph;
        int word, bit;

        KASSERT(mutex_owned(chan_mtx(periph->periph_channel)));

        SDT_PROBE3(scsi, base, tag, put,
            xs, xs->xs_tag_id, xs->xs_tag_type);

        word = xs->xs_tag_id >> 5;
        bit = xs->xs_tag_id & 0x1f;

        periph->periph_freetags[word] |= (1U << bit);
}

/*
 * scsipi_get_xs:
 *
 *      Allocate an xfer descriptor and associate it with the
 *      specified peripheral.  If the peripheral has no more
 *      available command openings, we either block waiting for
 *      one to become available, or fail.
 *
 *      When this routine is called with the channel lock held
 *      the flags must include XS_CTL_NOSLEEP.
 */
struct scsipi_xfer *
scsipi_get_xs(struct scsipi_periph *periph, int flags)
{
        struct scsipi_xfer *xs;
        bool lock = (flags & XS_CTL_NOSLEEP) == 0;

        SC_DEBUG(periph, SCSIPI_DB3, ("scsipi_get_xs\n"));

        KASSERT(!cold);

#ifdef DIAGNOSTIC
        /*
         * URGENT commands can never be ASYNC.
         */
        if ((flags & (XS_CTL_URGENT|XS_CTL_ASYNC)) ==
            (XS_CTL_URGENT|XS_CTL_ASYNC)) {
                scsipi_printaddr(periph);
                printf("URGENT and ASYNC\n");
                panic("scsipi_get_xs");
        }
#endif

        /*
         * Wait for a command opening to become available.  Rules:
         *
         *      - All xfers must wait for an available opening.
         *        Exception: URGENT xfers can proceed when
         *        active == openings, because we use the opening
         *        of the command we're recovering for.
         *      - if the periph has sense pending, only URGENT & REQSENSE
         *        xfers may proceed.
         *
         *      - If the periph is recovering, only URGENT xfers may
         *        proceed.
         *
         *      - If the periph is currently executing a recovery
         *        command, URGENT commands must block, because only
         *        one recovery command can execute at a time.
         */
        if (lock)
                mutex_enter(chan_mtx(periph->periph_channel));
        for (;;) {
                if (flags & XS_CTL_URGENT) {
                        if (periph->periph_active > periph->periph_openings)
                                goto wait_for_opening;
                        if (periph->periph_flags & PERIPH_SENSE) {
                                if ((flags & XS_CTL_REQSENSE) == 0)
                                        goto wait_for_opening;
                        } else {
                                if ((periph->periph_flags &
                                    PERIPH_RECOVERY_ACTIVE) != 0)
                                        goto wait_for_opening;
                                periph->periph_flags |= PERIPH_RECOVERY_ACTIVE;
                        }
                        break;
                }
                if (periph->periph_active >= periph->periph_openings ||
                    (periph->periph_flags & PERIPH_RECOVERING) != 0)
                        goto wait_for_opening;
                periph->periph_active++;
                KASSERT(mutex_owned(chan_mtx(periph->periph_channel)));
                break;

 wait_for_opening:
                if (flags & XS_CTL_NOSLEEP) {
                        KASSERT(!lock);
                        return NULL;
                }
                KASSERT(lock);
                SC_DEBUG(periph, SCSIPI_DB3, ("sleeping\n"));
                periph->periph_flags |= PERIPH_WAITING;
                cv_wait(periph_cv_periph(periph),
                    chan_mtx(periph->periph_channel));
        }
        if (lock)
                mutex_exit(chan_mtx(periph->periph_channel));

        SC_DEBUG(periph, SCSIPI_DB3, ("calling pool_get\n"));
        xs = pool_get(&scsipi_xfer_pool,
            ((flags & XS_CTL_NOSLEEP) != 0 ? PR_NOWAIT : PR_WAITOK));
        if (xs == NULL) {
                if (lock)
                        mutex_enter(chan_mtx(periph->periph_channel));
                if (flags & XS_CTL_URGENT) {
                        if ((flags & XS_CTL_REQSENSE) == 0)
                                periph->periph_flags &= ~PERIPH_RECOVERY_ACTIVE;
                } else
                        periph->periph_active--;
                if (lock)
                        mutex_exit(chan_mtx(periph->periph_channel));
                scsipi_printaddr(periph);
                printf("unable to allocate %sscsipi_xfer\n",
                    (flags & XS_CTL_URGENT) ? "URGENT " : "");
        }

        SC_DEBUG(periph, SCSIPI_DB3, ("returning\n"));

        if (xs != NULL) {
                memset(xs, 0, sizeof(*xs));
                callout_init(&xs->xs_callout, 0);
                xs->xs_periph = periph;
                xs->xs_control = flags;
                xs->xs_status = 0;
                if ((flags & XS_CTL_NOSLEEP) == 0)
                        mutex_enter(chan_mtx(periph->periph_channel));
                TAILQ_INSERT_TAIL(&periph->periph_xferq, xs, device_q);
                KASSERT(mutex_owned(chan_mtx(periph->periph_channel)));
                if ((flags & XS_CTL_NOSLEEP) == 0)
                        mutex_exit(chan_mtx(periph->periph_channel));
        }
        return xs;
}

/*
 * scsipi_put_xs:
 *
 *      Release an xfer descriptor, decreasing the outstanding command
 *      count for the peripheral.  If there is a thread waiting for
 *      an opening, wake it up.  If not, kick any queued I/O the
 *      peripheral may have.
 *
 *      NOTE: Must be called with channel lock held
 */
void
scsipi_put_xs(struct scsipi_xfer *xs)
{
        struct scsipi_periph *periph = xs->xs_periph;
        int flags = xs->xs_control;

        SDT_PROBE1(scsi, base, xfer, free,  xs);
        SC_DEBUG(periph, SCSIPI_DB3, ("scsipi_free_xs\n"));
        KASSERT(mutex_owned(chan_mtx(periph->periph_channel)));

        TAILQ_REMOVE(&periph->periph_xferq, xs, device_q);
        callout_destroy(&xs->xs_callout);
        pool_put(&scsipi_xfer_pool, xs);

#ifdef DIAGNOSTIC
        if ((periph->periph_flags & PERIPH_RECOVERY_ACTIVE) != 0 &&
            periph->periph_active == 0) {
                scsipi_printaddr(periph);
                printf("recovery without a command to recovery for\n");
                panic("scsipi_put_xs");
        }
#endif

        if (flags & XS_CTL_URGENT) {
                if ((flags & XS_CTL_REQSENSE) == 0)
                        periph->periph_flags &= ~PERIPH_RECOVERY_ACTIVE;
        } else
                periph->periph_active--;
        if (periph->periph_active == 0 &&
            (periph->periph_flags & PERIPH_WAITDRAIN) != 0) {
                periph->periph_flags &= ~PERIPH_WAITDRAIN;
                cv_broadcast(periph_cv_active(periph));
        }

        if (periph->periph_flags & PERIPH_WAITING) {
                periph->periph_flags &= ~PERIPH_WAITING;
                cv_broadcast(periph_cv_periph(periph));
        } else {
                if (periph->periph_switch->psw_start != NULL &&
                    device_is_active(periph->periph_dev)) {
                        SC_DEBUG(periph, SCSIPI_DB2,
                            ("calling private start()\n"));
                        (*periph->periph_switch->psw_start)(periph);
                }
        }
}

/*
 * scsipi_channel_freeze:
 *
 *      Freeze a channel's xfer queue.
 */
void
scsipi_channel_freeze(struct scsipi_channel *chan, int count)
{
        bool lock = chan_running(chan) > 0;

        if (lock)
                mutex_enter(chan_mtx(chan));
        chan->chan_qfreeze += count;
        if (lock)
                mutex_exit(chan_mtx(chan));
}

static void
scsipi_channel_freeze_locked(struct scsipi_channel *chan, int count)
{

        chan->chan_qfreeze += count;
}

/*
 * scsipi_channel_thaw:
 *
 *      Thaw a channel's xfer queue.
 */
void
scsipi_channel_thaw(struct scsipi_channel *chan, int count)
{
        bool lock = chan_running(chan) > 0;

        if (lock)
                mutex_enter(chan_mtx(chan));
        chan->chan_qfreeze -= count;
        /*
         * Don't let the freeze count go negative.
         *
         * Presumably the adapter driver could keep track of this,
         * but it might just be easier to do this here so as to allow
         * multiple callers, including those outside the adapter driver.
         */
        if (chan->chan_qfreeze < 0) {
                chan->chan_qfreeze = 0;
        }
        if (lock)
                mutex_exit(chan_mtx(chan));

        /*
         * until the channel is running
         */
        if (!lock)
                return;

        /*
         * Kick the channel's queue here.  Note, we may be running in
         * interrupt context (softclock or HBA's interrupt), so the adapter
         * driver had better not sleep.
         */
        if (chan->chan_qfreeze == 0)
                scsipi_run_queue(chan);
}

/*
 * scsipi_channel_timed_thaw:
 *
 *      Thaw a channel after some time has expired. This will also
 *      run the channel's queue if the freeze count has reached 0.
 */
void
scsipi_channel_timed_thaw(void *arg)
{
        struct scsipi_channel *chan = arg;

        scsipi_channel_thaw(chan, 1);
}

/*
 * scsipi_periph_freeze:
 *
 *      Freeze a device's xfer queue.
 */
void
scsipi_periph_freeze_locked(struct scsipi_periph *periph, int count)
{

        periph->periph_qfreeze += count;
}

/*
 * scsipi_periph_thaw:
 *
 *      Thaw a device's xfer queue.
 */
void
scsipi_periph_thaw_locked(struct scsipi_periph *periph, int count)
{

        periph->periph_qfreeze -= count;
#ifdef DIAGNOSTIC
        if (periph->periph_qfreeze < 0) {
                static const char pc[] = "periph freeze count < 0";
                scsipi_printaddr(periph);
                printf("%s\n", pc);
                panic(pc);
        }
#endif
        if (periph->periph_qfreeze == 0 &&
            (periph->periph_flags & PERIPH_WAITING) != 0)
                cv_broadcast(periph_cv_periph(periph));
}

void
scsipi_periph_freeze(struct scsipi_periph *periph, int count)
{

        mutex_enter(chan_mtx(periph->periph_channel));
        scsipi_periph_freeze_locked(periph, count);
        mutex_exit(chan_mtx(periph->periph_channel));
}

void
scsipi_periph_thaw(struct scsipi_periph *periph, int count)
{

        mutex_enter(chan_mtx(periph->periph_channel));
        scsipi_periph_thaw_locked(periph, count);
        mutex_exit(chan_mtx(periph->periph_channel));
}

/*
 * scsipi_periph_timed_thaw:
 *
 *      Thaw a device after some time has expired.
 */
void
scsipi_periph_timed_thaw(void *arg)
{
        struct scsipi_periph *periph = arg;
        struct scsipi_channel *chan = periph->periph_channel;

        callout_stop(&periph->periph_callout);

        mutex_enter(chan_mtx(chan));
        scsipi_periph_thaw_locked(periph, 1);
        if ((periph->periph_channel->chan_flags & SCSIPI_CHAN_TACTIVE) == 0) {
                /*
                 * Kick the channel's queue here.  Note, we're running in
                 * interrupt context (softclock), so the adapter driver
                 * had better not sleep.
                 */
                mutex_exit(chan_mtx(chan));
                scsipi_run_queue(periph->periph_channel);
        } else {
                /*
                 * Tell the completion thread to kick the channel's queue here.
                 */
                periph->periph_channel->chan_tflags |= SCSIPI_CHANT_KICK;
                cv_broadcast(chan_cv_complete(chan));
                mutex_exit(chan_mtx(chan));
        }
}

/*
 * scsipi_wait_drain:
 *
 *      Wait for a periph's pending xfers to drain.
 */
void
scsipi_wait_drain(struct scsipi_periph *periph)
{
        struct scsipi_channel *chan = periph->periph_channel;

        mutex_enter(chan_mtx(chan));
        while (periph->periph_active != 0) {
                periph->periph_flags |= PERIPH_WAITDRAIN;
                cv_wait(periph_cv_active(periph), chan_mtx(chan));
        }
        mutex_exit(chan_mtx(chan));
}

/*
 * scsipi_kill_pending:
 *
 *      Kill off all pending xfers for a periph.
 *
 *      NOTE: Must be called with channel lock held
 */
void
scsipi_kill_pending(struct scsipi_periph *periph)
{
        struct scsipi_channel *chan = periph->periph_channel;

        (*chan->chan_bustype->bustype_kill_pending)(periph);
        while (periph->periph_active != 0) {
                periph->periph_flags |= PERIPH_WAITDRAIN;
                cv_wait(periph_cv_active(periph), chan_mtx(chan));
        }
}

/*
 * scsipi_print_cdb:
 * prints a command descriptor block (for debug purpose, error messages,
 * SCSIVERBOSE, ...)
 */
void
scsipi_print_cdb(struct scsipi_generic *cmd)
{
        int i, j;

        printf("0x%02x", cmd->opcode);

        switch (CDB_GROUPID(cmd->opcode)) {
        case CDB_GROUPID_0:
                j = CDB_GROUP0;
                break;
        case CDB_GROUPID_1:
                j = CDB_GROUP1;
                break;
        case CDB_GROUPID_2:
                j = CDB_GROUP2;
                break;
        case CDB_GROUPID_3:
                j = CDB_GROUP3;
                break;
        case CDB_GROUPID_4:
                j = CDB_GROUP4;
                break;
        case CDB_GROUPID_5:
                j = CDB_GROUP5;
                break;
        case CDB_GROUPID_6:
                j = CDB_GROUP6;
                break;
        case CDB_GROUPID_7:
                j = CDB_GROUP7;
                break;
        default:
                j = 0;
        }
        if (j == 0)
                j = sizeof (cmd->bytes);
        for (i = 0; i < j-1; i++) /* already done the opcode */
                printf(" %02x", cmd->bytes[i]);
}

/*
 * scsipi_interpret_sense:
 *
 *      Look at the returned sense and act on the error, determining
 *      the unix error number to pass back.  (0 = report no error)
 *
 *      NOTE: If we return ERESTART, we are expected to have
 *      thawed the device!
 *
 *      THIS IS THE DEFAULT ERROR HANDLER FOR SCSI DEVICES.
 */
int
scsipi_interpret_sense(struct scsipi_xfer *xs)
{
        struct scsi_sense_data *sense;
        struct scsipi_periph *periph = xs->xs_periph;
        u_int8_t key;
        int error;
        u_int32_t info;
        static const char *error_mes[] = {
                "soft error (corrected)",
                "not ready", "medium error",
                "non-media hardware failure", "illegal request",
                "unit attention", "readonly device",
                "no data found", "vendor unique",
                "copy aborted", "command aborted",
                "search returned equal", "volume overflow",
                "verify miscompare", "unknown error key"
        };

        sense = &xs->sense.scsi_sense;
#ifdef SCSIPI_DEBUG
        if (periph->periph_flags & SCSIPI_DB1) {
                int count, len;
                scsipi_printaddr(periph);
                printf(" sense debug information:\n");
                printf("\tcode 0x%x valid %d\n",
                        SSD_RCODE(sense->response_code),
                        sense->response_code & SSD_RCODE_VALID ? 1 : 0);
                printf("\tseg 0x%x key 0x%x ili 0x%x eom 0x%x fmark 0x%x\n",
                        sense->segment,
                        SSD_SENSE_KEY(sense->flags),
                        sense->flags & SSD_ILI ? 1 : 0,
                        sense->flags & SSD_EOM ? 1 : 0,
                        sense->flags & SSD_FILEMARK ? 1 : 0);
                printf("\ninfo: 0x%x 0x%x 0x%x 0x%x followed by %d "
                        "extra bytes\n",
                        sense->info[0],
                        sense->info[1],
                        sense->info[2],
                        sense->info[3],
                        sense->extra_len);
                len = SSD_ADD_BYTES_LIM(sense);
                printf("\textra (up to %d bytes): ", len);
                for (count = 0; count < len; count++)
                        printf("0x%x ", sense->csi[count]);
                printf("\n");
        }
#endif

        /*
         * If the periph has its own error handler, call it first.
         * If it returns a legit error value, return that, otherwise
         * it wants us to continue with normal error processing.
         */
        if (periph->periph_switch->psw_error != NULL) {
                SC_DEBUG(periph, SCSIPI_DB2,
                    ("calling private err_handler()\n"));
                error = (*periph->periph_switch->psw_error)(xs);
                if (error != EJUSTRETURN)
                        return error;
        }
        /* otherwise use the default */
        switch (SSD_RCODE(sense->response_code)) {

                /*
                 * Old SCSI-1 and SASI devices respond with
                 * codes other than 70.
                 */
        case 0x00:              /* no error (command completed OK) */
                return 0;
        case 0x04:              /* drive not ready after it was selected */
                if ((periph->periph_flags & PERIPH_REMOVABLE) != 0)
                        periph->periph_flags &= ~PERIPH_MEDIA_LOADED;
                if ((xs->xs_control & XS_CTL_IGNORE_NOT_READY) != 0)
                        return 0;
                /* XXX - display some sort of error here? */
                return EIO;
        case 0x20:              /* invalid command */
                if ((xs->xs_control &
                     XS_CTL_IGNORE_ILLEGAL_REQUEST) != 0)
                        return 0;
                return EINVAL;
        case 0x25:              /* invalid LUN (Adaptec ACB-4000) */
                return EACCES;

                /*
                 * If it's code 70, use the extended stuff and
                 * interpret the key
                 */
        case 0x71:              /* delayed error */
                scsipi_printaddr(periph);
                key = SSD_SENSE_KEY(sense->flags);
                printf(" DEFERRED ERROR, key = 0x%x\n", key);
                /* FALLTHROUGH */
        case 0x70:
                if ((sense->response_code & SSD_RCODE_VALID) != 0)
                        info = _4btol(sense->info);
                else
                        info = 0;
                key = SSD_SENSE_KEY(sense->flags);

                switch (key) {
                case SKEY_NO_SENSE:
                case SKEY_RECOVERED_ERROR:
                        if (xs->resid == xs->datalen && xs->datalen) {
                                /*
                                 * Why is this here?
                                 */
                                xs->resid = 0;  /* not short read */
                        }
                        error = 0;
                        break;
                case SKEY_EQUAL:
                        error = 0;
                        break;
                case SKEY_NOT_READY:
                        if ((periph->periph_flags & PERIPH_REMOVABLE) != 0)
                                periph->periph_flags &= ~PERIPH_MEDIA_LOADED;
                        if ((xs->xs_control & XS_CTL_IGNORE_NOT_READY) != 0)
                                return 0;
                        if (sense->asc == 0x3A) {
                                error = ENODEV; /* Medium not present */
                                if (xs->xs_control & XS_CTL_SILENT_NODEV)
                                        return error;
                        } else
                                error = EIO;
                        if ((xs->xs_control & XS_CTL_SILENT) != 0)
                                return error;
                        break;
                case SKEY_ILLEGAL_REQUEST:
                        if ((xs->xs_control &
                             XS_CTL_IGNORE_ILLEGAL_REQUEST) != 0)
                                return 0;
                        /*
                         * Handle the case where a device reports
                         * Logical Unit Not Supported during discovery.
                         */
                        if ((xs->xs_control & XS_CTL_DISCOVERY) != 0 &&
                            sense->asc == 0x25 &&
                            sense->ascq == 0x00)
                                return EINVAL;
                        if ((xs->xs_control & XS_CTL_SILENT) != 0)
                                return EIO;
                        error = EINVAL;
                        break;
                case SKEY_UNIT_ATTENTION:
                        if (sense->asc == 0x29 &&
                            sense->ascq == 0x00) {
                                /* device or bus reset */
                                return ERESTART;
                        }
                        if ((periph->periph_flags & PERIPH_REMOVABLE) != 0)
                                periph->periph_flags &= ~PERIPH_MEDIA_LOADED;
                        if ((xs->xs_control &
                             XS_CTL_IGNORE_MEDIA_CHANGE) != 0 ||
                                /* XXX Should reupload any transient state. */
                                (periph->periph_flags &
                                 PERIPH_REMOVABLE) == 0) {
                                return ERESTART;
                        }
                        if ((xs->xs_control & XS_CTL_SILENT) != 0)
                                return EIO;
                        error = EIO;
                        break;
                case SKEY_DATA_PROTECT:
                        error = EROFS;
                        break;
                case SKEY_BLANK_CHECK:
                        error = 0;
                        break;
                case SKEY_ABORTED_COMMAND:
                        if (xs->xs_retries != 0) {
                                xs->xs_retries--;
                                error = ERESTART;
                        } else
                                error = EIO;
                        break;
                case SKEY_VOLUME_OVERFLOW:
                        error = ENOSPC;
                        break;
                case SKEY_MEDIUM_ERROR:
                        if (xs->xs_retries != 0) {
                                xs->xs_retries--;
                                error = ERESTART;
                        } else
                                error = EIO;
                        break;
                default:
                        error = EIO;
                        break;
                }

                /* Print verbose decode if appropriate and possible */
                if ((key == 0) ||
                    ((xs->xs_control & XS_CTL_SILENT) != 0) ||
                    (scsipi_print_sense(xs, 0) != 0))
                        return error;

                /* Print brief(er) sense information */
                scsipi_printaddr(periph);
                printf("%s", error_mes[key - 1]);
                if ((sense->response_code & SSD_RCODE_VALID) != 0) {
                        switch (key) {
                        case SKEY_NOT_READY:
                        case SKEY_ILLEGAL_REQUEST:
                        case SKEY_UNIT_ATTENTION:
                        case SKEY_DATA_PROTECT:
                                break;
                        case SKEY_BLANK_CHECK:
                                printf(", requested size: %d (decimal)",
                                    info);
                                break;
                        case SKEY_ABORTED_COMMAND:
                                if (xs->xs_retries)
                                        printf(", retrying");
                                printf(", cmd 0x%x, info 0x%x",
                                    xs->cmd->opcode, info);
                                break;
                        default:
                                printf(", info = %d (decimal)", info);
                        }
                }
                if (sense->extra_len != 0) {
                        int n;
                        printf(", data =");
                        for (n = 0; n < sense->extra_len; n++)
                                printf(" %02x",
                                    sense->csi[n]);
                }
                printf("\n");
                return error;

        /*
         * Some other code, just report it
         */
        default:
#if    defined(SCSIDEBUG) || defined(DEBUG)
        {
                static const char *uc = "undecodable sense error";
                int i;
                u_int8_t *cptr = (u_int8_t *) sense;
                scsipi_printaddr(periph);
                if (xs->cmd == &xs->cmdstore) {
                        printf("%s for opcode 0x%x, data=",
                            uc, xs->cmdstore.opcode);
                } else {
                        printf("%s, data=", uc);
                }
                for (i = 0; i < sizeof (sense); i++)
                        printf(" 0x%02x", *(cptr++) & 0xff);
                printf("\n");
        }
#else
                scsipi_printaddr(periph);
                printf("Sense Error Code 0x%x",
                        SSD_RCODE(sense->response_code));
                if ((sense->response_code & SSD_RCODE_VALID) != 0) {
                        struct scsi_sense_data_unextended *usense =
                            (struct scsi_sense_data_unextended *)sense;
                        printf(" at block no. %d (decimal)",
                            _3btol(usense->block));
                }
                printf("\n");
#endif
                return EIO;
        }
}

/*
 * scsipi_test_unit_ready:
 *
 *      Issue a `test unit ready' request.
 */
int
scsipi_test_unit_ready(struct scsipi_periph *periph, int flags)
{
        struct scsi_test_unit_ready cmd;
        int retries;

        /* some ATAPI drives don't support TEST UNIT READY. Sigh */
        if (periph->periph_quirks & PQUIRK_NOTUR)
                return 0;

        if (flags & XS_CTL_DISCOVERY)
                retries = 0;
        else
                retries = SCSIPIRETRIES;

        memset(&cmd, 0, sizeof(cmd));
        cmd.opcode = SCSI_TEST_UNIT_READY;

        return scsipi_command(periph, (void *)&cmd, sizeof(cmd), 0, 0,
            retries, 10000, NULL, flags);
}

static const struct scsipi_inquiry3_pattern {
        const char vendor[8];
        const char product[16];
        const char revision[4];
} scsipi_inquiry3_quirk[] = {
        { "ES-6600 ", "", "" },
};

static int
scsipi_inquiry3_ok(const struct scsipi_inquiry_data *ib)
{
        for (size_t i = 0; i < __arraycount(scsipi_inquiry3_quirk); i++) {
                const struct scsipi_inquiry3_pattern *q =
                    &scsipi_inquiry3_quirk[i];
#define MATCH(field) \
    (q->field[0] ? memcmp(ib->field, q->field, sizeof(ib->field)) == 0 : 1)
                if (MATCH(vendor) && MATCH(product) && MATCH(revision))
                        return 0;
        }
        return 1;
}

/*
 * scsipi_inquire:
 *
 *      Ask the device about itself.
 */
int
scsipi_inquire(struct scsipi_periph *periph, struct scsipi_inquiry_data *inqbuf,
    int flags)
{
        struct scsipi_inquiry cmd;
        int error;
        int retries;

        if (flags & XS_CTL_DISCOVERY)
                retries = 0;
        else
                retries = SCSIPIRETRIES;

        /*
         * If we request more data than the device can provide, it SHOULD just
         * return a short response.  However, some devices error with an
         * ILLEGAL REQUEST sense code, and yet others have even more special
         * failure modes (such as the GL641USB flash adapter, which goes loony
         * and sends corrupted CRCs).  To work around this, and to bring our
         * behavior more in line with other OSes, we do a shorter inquiry,
         * covering all the SCSI-2 information, first, and then request more
         * data iff the "additional length" field indicates there is more.
         * - mycroft, 2003/10/16
         */
        memset(&cmd, 0, sizeof(cmd));
        cmd.opcode = INQUIRY;
        cmd.length = SCSIPI_INQUIRY_LENGTH_SCSI2;
        error = scsipi_command(periph, (void *)&cmd, sizeof(cmd),
            (void *)inqbuf, SCSIPI_INQUIRY_LENGTH_SCSI2, retries,
            10000, NULL, flags | XS_CTL_DATA_IN);
        if (!error &&
            inqbuf->additional_length > SCSIPI_INQUIRY_LENGTH_SCSI2 - 4) {
            if (scsipi_inquiry3_ok(inqbuf)) {
#if 0
printf("inquire: addlen=%d, retrying\n", inqbuf->additional_length);
#endif
                cmd.length = SCSIPI_INQUIRY_LENGTH_SCSI3;
                error = scsipi_command(periph, (void *)&cmd, sizeof(cmd),
                    (void *)inqbuf, SCSIPI_INQUIRY_LENGTH_SCSI3, retries,
                    10000, NULL, flags | XS_CTL_DATA_IN);
#if 0
printf("inquire: error=%d\n", error);
#endif
            }
        }

#ifdef SCSI_OLD_NOINQUIRY
        /*
         * Kludge for the Adaptec ACB-4000 SCSI->MFM translator.
         * This board doesn't support the INQUIRY command at all.
         */
        if (error == EINVAL || error == EACCES) {
                /*
                 * Conjure up an INQUIRY response.
                 */
                inqbuf->device = (error == EINVAL ?
                         SID_QUAL_LU_PRESENT :
                         SID_QUAL_LU_NOTPRESENT) | T_DIRECT;
                inqbuf->dev_qual2 = 0;
                inqbuf->version = 0;
                inqbuf->response_format = SID_FORMAT_SCSI1;
                inqbuf->additional_length = SCSIPI_INQUIRY_LENGTH_SCSI2 - 4;
                inqbuf->flags1 = inqbuf->flags2 = inqbuf->flags3 = 0;
                memcpy(inqbuf->vendor, "ADAPTEC ACB-4000            ", 28);
                error = 0;
        }

        /*
         * Kludge for the Emulex MT-02 SCSI->QIC translator.
         * This board gives an empty response to an INQUIRY command.
         */
        else if (error == 0 &&
            inqbuf->device == (SID_QUAL_LU_PRESENT | T_DIRECT) &&
            inqbuf->dev_qual2 == 0 &&
            inqbuf->version == 0 &&
            inqbuf->response_format == SID_FORMAT_SCSI1) {
                /*
                 * Fill out the INQUIRY response.
                 */
                inqbuf->device = (SID_QUAL_LU_PRESENT | T_SEQUENTIAL);
                inqbuf->dev_qual2 = SID_REMOVABLE;
                inqbuf->additional_length = SCSIPI_INQUIRY_LENGTH_SCSI2 - 4;
                inqbuf->flags1 = inqbuf->flags2 = inqbuf->flags3 = 0;
                memcpy(inqbuf->vendor, "EMULEX  MT-02 QIC           ", 28);
        }
#endif /* SCSI_OLD_NOINQUIRY */

        return error;
}

/*
 * scsipi_prevent:
 *
 *      Prevent or allow the user to remove the media
 */
int
scsipi_prevent(struct scsipi_periph *periph, int type, int flags)
{
        struct scsi_prevent_allow_medium_removal cmd;

        if (periph->periph_quirks & PQUIRK_NODOORLOCK)
                return 0;

        memset(&cmd, 0, sizeof(cmd));
        cmd.opcode = SCSI_PREVENT_ALLOW_MEDIUM_REMOVAL;
        cmd.how = type;

        return (scsipi_command(periph, (void *)&cmd, sizeof(cmd), 0, 0,
            SCSIPIRETRIES, 5000, NULL, flags));
}

/*
 * scsipi_start:
 *
 *      Send a START UNIT.
 */
int
scsipi_start(struct scsipi_periph *periph, int type, int flags)
{
        struct scsipi_start_stop cmd;

        memset(&cmd, 0, sizeof(cmd));
        cmd.opcode = START_STOP;
        cmd.byte2 = 0x00;
        cmd.how = type;

        return scsipi_command(periph, (void *)&cmd, sizeof(cmd), 0, 0,
            SCSIPIRETRIES, (type & SSS_START) ? 60000 : 10000, NULL, flags);
}

/*
 * scsipi_mode_sense, scsipi_mode_sense_big:
 *      get a sense page from a device
 */

int
scsipi_mode_sense(struct scsipi_periph *periph, int byte2, int page,
    struct scsi_mode_parameter_header_6 *data, int len, int flags, int retries,
    int timeout)
{
        struct scsi_mode_sense_6 cmd;

        memset(&cmd, 0, sizeof(cmd));
        cmd.opcode = SCSI_MODE_SENSE_6;
        cmd.byte2 = byte2;
        cmd.page = page;
        cmd.length = len & 0xff;

        return scsipi_command(periph, (void *)&cmd, sizeof(cmd),
            (void *)data, len, retries, timeout, NULL, flags | XS_CTL_DATA_IN);
}

int
scsipi_mode_sense_big(struct scsipi_periph *periph, int byte2, int page,
    struct scsi_mode_parameter_header_10 *data, int len, int flags, int retries,
    int timeout)
{
        struct scsi_mode_sense_10 cmd;

        memset(&cmd, 0, sizeof(cmd));
        cmd.opcode = SCSI_MODE_SENSE_10;
        cmd.byte2 = byte2;
        cmd.page = page;
        _lto2b(len, cmd.length);

        return scsipi_command(periph, (void *)&cmd, sizeof(cmd),
            (void *)data, len, retries, timeout, NULL, flags | XS_CTL_DATA_IN);
}

int
scsipi_mode_select(struct scsipi_periph *periph, int byte2,
    struct scsi_mode_parameter_header_6 *data, int len, int flags, int retries,
    int timeout)
{
        struct scsi_mode_select_6 cmd;

        memset(&cmd, 0, sizeof(cmd));
        cmd.opcode = SCSI_MODE_SELECT_6;
        cmd.byte2 = byte2;
        cmd.length = len & 0xff;

        return scsipi_command(periph, (void *)&cmd, sizeof(cmd),
            (void *)data, len, retries, timeout, NULL, flags | XS_CTL_DATA_OUT);
}

int
scsipi_mode_select_big(struct scsipi_periph *periph, int byte2,
    struct scsi_mode_parameter_header_10 *data, int len, int flags, int retries,
    int timeout)
{
        struct scsi_mode_select_10 cmd;

        memset(&cmd, 0, sizeof(cmd));
        cmd.opcode = SCSI_MODE_SELECT_10;
        cmd.byte2 = byte2;
        _lto2b(len, cmd.length);

        return scsipi_command(periph, (void *)&cmd, sizeof(cmd),
            (void *)data, len, retries, timeout, NULL, flags | XS_CTL_DATA_OUT);
}

/*
 * scsipi_get_opcodeinfo:
 *
 * query the device for supported commands and their timeout
 * building a timeout lookup table if timeout information is available.
 */
void
scsipi_get_opcodeinfo(struct scsipi_periph *periph)
{
        u_int8_t *data;
        int len = 16*1024;
        int rc;
        int retries;
        struct scsi_repsuppopcode cmd;
        
        /* refrain from asking for supported opcodes */
        if (periph->periph_quirks & PQUIRK_NOREPSUPPOPC ||
            periph->periph_type == T_PROCESSOR || /* spec. */
            periph->periph_type == T_CDROM) /* spec. */
                return;

        scsipi_free_opcodeinfo(periph);

        /*
         * query REPORT SUPPORTED OPERATION CODES
         * if OK
         *   enumerate all codes
         *     if timeout exists insert maximum into opcode table
         */
        data = malloc(len, M_DEVBUF, M_WAITOK|M_ZERO);

        memset(&cmd, 0, sizeof(cmd));
        cmd.opcode = SCSI_MAINTENANCE_IN;
        cmd.svcaction = RSOC_REPORT_SUPPORTED_OPCODES;
        cmd.repoption = RSOC_RCTD|RSOC_ALL;
        _lto4b(len, cmd.alloclen);
        
        /* loop to skip any UNIT ATTENTIONS at this point */
        retries = 3;
        do {
                rc = scsipi_command(periph, (void *)&cmd, sizeof(cmd),
                                    (void *)data, len, 0, 60000, NULL,
                                    XS_CTL_DATA_IN|XS_CTL_SILENT);
#ifdef SCSIPI_DEBUG
                if (rc != 0) {
                        SC_DEBUG(periph, SCSIPI_DB3,
                                ("SCSI_MAINTENANCE_IN"
                                "[RSOC_REPORT_SUPPORTED_OPCODES] command"
                                " failed: rc=%d, retries=%d\n",
                                rc, retries));
                }
#endif
        } while (rc == EIO && retries-- > 0);

        if (rc == 0) {
                int count;
                int dlen = _4btol(data);
                u_int8_t *c = data + 4;
                
                SC_DEBUG(periph, SCSIPI_DB3,
                         ("supported opcode timeout-values loaded\n"));
                SC_DEBUG(periph, SCSIPI_DB3,
                         ("CMD  LEN  SA    spec  nom. time  cmd timeout\n"));

                struct scsipi_opcodes *tot = malloc(sizeof(struct scsipi_opcodes),
                    M_DEVBUF, M_WAITOK|M_ZERO);

                count = 0;
                while (tot != NULL &&
                       dlen >= (int)sizeof(struct scsi_repsupopcode_all_commands_descriptor)) {
                        struct scsi_repsupopcode_all_commands_descriptor *acd
                                = (struct scsi_repsupopcode_all_commands_descriptor *)c;
#ifdef SCSIPI_DEBUG
                        int cdblen = _2btol((const u_int8_t *)&acd->cdblen);
#endif
                        dlen -= sizeof(struct scsi_repsupopcode_all_commands_descriptor);
                        c += sizeof(struct scsi_repsupopcode_all_commands_descriptor);
                        SC_DEBUG(periph, SCSIPI_DB3,
                                 ("0x%02x(%2d) ", acd->opcode, cdblen));
                        
                        tot->opcode_info[acd->opcode].ti_flags = SCSIPI_TI_VALID;
                        
                        if (acd->flags & RSOC_ACD_SERVACTV) {
                                SC_DEBUGN(periph, SCSIPI_DB3,
                                         ("0x%02x%02x ",
                                          acd->serviceaction[0],
                                          acd->serviceaction[1]));
                        } else {
                                SC_DEBUGN(periph, SCSIPI_DB3, ("       "));
                        }
                        
                        if (acd->flags & RSOC_ACD_CTDP
                            && dlen >= (int)sizeof(struct scsi_repsupopcode_timeouts_descriptor)) {
                                struct scsi_repsupopcode_timeouts_descriptor *td
                                        = (struct scsi_repsupopcode_timeouts_descriptor *)c;
                                long nomto = _4btol(td->nom_process_timeout);
                                long cmdto = _4btol(td->cmd_process_timeout);
                                long t = (cmdto > nomto) ? cmdto : nomto;

                                dlen -= sizeof(struct scsi_repsupopcode_timeouts_descriptor);
                                c += sizeof(struct scsi_repsupopcode_timeouts_descriptor);

                                SC_DEBUGN(periph, SCSIPI_DB3,
                                          ("0x%02x %10ld %10ld",
                                           td->cmd_specific,
                                           nomto, cmdto));

                                if (t > tot->opcode_info[acd->opcode].ti_timeout) {
                                        tot->opcode_info[acd->opcode].ti_timeout = t;
                                        ++count;
                                }
                        }
                        SC_DEBUGN(periph, SCSIPI_DB3,("\n"));
                }

                if (count > 0) {
                        periph->periph_opcs = tot;
                } else {
                        free(tot, M_DEVBUF);
                        SC_DEBUG(periph, SCSIPI_DB3,
                                ("no usable timeout values available\n"));
                }
        } else {
                SC_DEBUG(periph, SCSIPI_DB3,
                         ("SCSI_MAINTENANCE_IN"
                          "[RSOC_REPORT_SUPPORTED_OPCODES] failed error=%d"
                          " - no device provided timeout "
                          "values available\n", rc));
        }

        free(data, M_DEVBUF);
}

/*
 * scsipi_update_timeouts:
 *      Override timeout value if device/config provided
 *      timeouts are available.
 */
static void
scsipi_update_timeouts(struct scsipi_xfer *xs)
{
        struct scsipi_opcodes *opcs;
        u_int8_t cmd;
        int timeout;
        struct scsipi_opinfo *oi;
        
        if (xs->timeout <= 0) {
                return; 
        }
        
        opcs = xs->xs_periph->periph_opcs;
        
        if (opcs == NULL) {
                return;
        }
        
        cmd = xs->cmd->opcode;
        oi = &opcs->opcode_info[cmd];
        
        timeout = 1000 * (int)oi->ti_timeout;


        if (timeout > xs->timeout && timeout < 86400000) {
                /*
                 * pick up device configured timeouts if they
                 * are longer than the requested ones but less
                 * than a day
                 */
#ifdef SCSIPI_DEBUG
                if ((oi->ti_flags & SCSIPI_TI_LOGGED) == 0) {
                        SC_DEBUG(xs->xs_periph, SCSIPI_DB3,
                                 ("Overriding command 0x%02x "
                                  "timeout of %d with %d ms\n",
                                  cmd, xs->timeout, timeout));
                        oi->ti_flags |= SCSIPI_TI_LOGGED;
                }
#endif
                xs->timeout = timeout;
        }
}

/*
 * scsipi_free_opcodeinfo:
 *
 * free the opcode information table
 */
void
scsipi_free_opcodeinfo(struct scsipi_periph *periph)
{
        if (periph->periph_opcs != NULL) {
                free(periph->periph_opcs, M_DEVBUF);
        }

        periph->periph_opcs = NULL;
}

/*
 * scsipi_done:
 *
 *      This routine is called by an adapter's interrupt handler when
 *      an xfer is completed.
 */
void
scsipi_done(struct scsipi_xfer *xs)
{
        struct scsipi_periph *periph = xs->xs_periph;
        struct scsipi_channel *chan = periph->periph_channel;
        int freezecnt;

        SC_DEBUG(periph, SCSIPI_DB2, ("scsipi_done\n"));
#ifdef SCSIPI_DEBUG
        if (periph->periph_dbflags & SCSIPI_DB1)
                show_scsipi_cmd(xs);
#endif

        mutex_enter(chan_mtx(chan));
        SDT_PROBE1(scsi, base, xfer, done,  xs);
        /*
         * The resource this command was using is now free.
         */
        if (xs->xs_status & XS_STS_DONE) {
                /* XXX in certain circumstances, such as a device
                 * being detached, a xs that has already been
                 * scsipi_done()'d by the main thread will be done'd
                 * again by scsibusdetach(). Putting the xs on the
                 * chan_complete queue causes list corruption and
                 * everyone dies. This prevents that, but perhaps
                 * there should be better coordination somewhere such
                 * that this won't ever happen (and can be turned into
                 * a KASSERT().
                 */
                SDT_PROBE1(scsi, base, xfer, redone,  xs);
                mutex_exit(chan_mtx(chan));
                goto out;
        }
        scsipi_put_resource(chan);
        xs->xs_periph->periph_sent--;

        /*
         * If the command was tagged, free the tag.
         */
        if (XS_CTL_TAGTYPE(xs) != 0)
                scsipi_put_tag(xs);
        else
                periph->periph_flags &= ~PERIPH_UNTAG;

        /* Mark the command as `done'. */
        xs->xs_status |= XS_STS_DONE;

#ifdef DIAGNOSTIC
        if ((xs->xs_control & (XS_CTL_ASYNC|XS_CTL_POLL)) ==
            (XS_CTL_ASYNC|XS_CTL_POLL))
                panic("scsipi_done: ASYNC and POLL");
#endif

        /*
         * If the xfer had an error of any sort, freeze the
         * periph's queue.  Freeze it again if we were requested
         * to do so in the xfer.
         */
        freezecnt = 0;
        if (xs->error != XS_NOERROR)
                freezecnt++;
        if (xs->xs_control & XS_CTL_FREEZE_PERIPH)
                freezecnt++;
        if (freezecnt != 0)
                scsipi_periph_freeze_locked(periph, freezecnt);

        /*
         * record the xfer with a pending sense, in case a SCSI reset is
         * received before the thread is waked up.
         */
        if (xs->error == XS_BUSY && xs->status == SCSI_CHECK) {
                periph->periph_flags |= PERIPH_SENSE;
                periph->periph_xscheck = xs;
        }

        /*
         * If this was an xfer that was not to complete asynchronously,
         * let the requesting thread perform error checking/handling
         * in its context.
         */
        if ((xs->xs_control & XS_CTL_ASYNC) == 0) {
                /*
                 * If it's a polling job, just return, to unwind the
                 * call graph.  We don't need to restart the queue,
                 * because polling jobs are treated specially, and
                 * are really only used during crash dumps anyway
                 * (XXX or during boot-time autoconfiguration of
                 * ATAPI devices).
                 */
                if (xs->xs_control & XS_CTL_POLL) {
                        mutex_exit(chan_mtx(chan));
                        return;
                }
                cv_broadcast(xs_cv(xs));
                mutex_exit(chan_mtx(chan));
                goto out;
        }

        /*
         * Catch the extremely common case of I/O completing
         * without error; no use in taking a context switch
         * if we can handle it in interrupt context.
         */
        if (xs->error == XS_NOERROR) {
                mutex_exit(chan_mtx(chan));
                (void) scsipi_complete(xs);
                goto out;
        }

        /*
         * There is an error on this xfer.  Put it on the channel's
         * completion queue, and wake up the completion thread.
         */
        TAILQ_INSERT_TAIL(&chan->chan_complete, xs, channel_q);
        cv_broadcast(chan_cv_complete(chan));
        mutex_exit(chan_mtx(chan));

 out:
        /*
         * If there are more xfers on the channel's queue, attempt to
         * run them.
         */
        scsipi_run_queue(chan);
}

/*
 * scsipi_complete:
 *
 *      Completion of a scsipi_xfer.  This is the guts of scsipi_done().
 *
 *      NOTE: This routine MUST be called with valid thread context
 *      except for the case where the following two conditions are
 *      true:
 *
 *              xs->error == XS_NOERROR
 *              XS_CTL_ASYNC is set in xs->xs_control
 *
 *      The semantics of this routine can be tricky, so here is an
 *      explanation:
 *
 *              0               Xfer completed successfully.
 *
 *              ERESTART        Xfer had an error, but was restarted.
 *
 *              anything else   Xfer had an error, return value is Unix
 *                              errno.
 *
 *      If the return value is anything but ERESTART:
 *
 *              - If XS_CTL_ASYNC is set, `xs' has been freed back to
 *                the pool.
 *              - If there is a buf associated with the xfer,
 *                it has been biodone()'d.
 */
static int
scsipi_complete(struct scsipi_xfer *xs)
{
        struct scsipi_periph *periph = xs->xs_periph;
        struct scsipi_channel *chan = periph->periph_channel;
        int error;

        SDT_PROBE1(scsi, base, xfer, complete,  xs);

#ifdef DIAGNOSTIC
        if ((xs->xs_control & XS_CTL_ASYNC) != 0 && xs->bp == NULL)
                panic("scsipi_complete: XS_CTL_ASYNC but no buf");
#endif
        /*
         * If command terminated with a CHECK CONDITION, we need to issue a
         * REQUEST_SENSE command. Once the REQUEST_SENSE has been processed
         * we'll have the real status.
         * Must be processed with channel lock held to avoid missing
         * a SCSI bus reset for this command.
         */
        mutex_enter(chan_mtx(chan));
        if (xs->error == XS_BUSY && xs->status == SCSI_CHECK) {
                /* request sense for a request sense ? */
                if (xs->xs_control & XS_CTL_REQSENSE) {
                        scsipi_printaddr(periph);
                        printf("request sense for a request sense ?\n");
                        /* XXX maybe we should reset the device ? */
                        /* we've been frozen because xs->error != XS_NOERROR */
                        scsipi_periph_thaw_locked(periph, 1);
                        mutex_exit(chan_mtx(chan));
                        if (xs->resid < xs->datalen) {
                                printf("we read %d bytes of sense anyway:\n",
                                    xs->datalen - xs->resid);
                                scsipi_print_sense_data((void *)xs->data, 0);
                        }
                        return EINVAL;
                }
                mutex_exit(chan_mtx(chan)); // XXX allows other commands to queue or run
                scsipi_request_sense(xs);
        } else
                mutex_exit(chan_mtx(chan));

        /*
         * If it's a user level request, bypass all usual completion
         * processing, let the user work it out..
         */
        if ((xs->xs_control & XS_CTL_USERCMD) != 0) {
                SC_DEBUG(periph, SCSIPI_DB3, ("calling user done()\n"));
                mutex_enter(chan_mtx(chan));
                if (xs->error != XS_NOERROR)
                        scsipi_periph_thaw_locked(periph, 1);
                mutex_exit(chan_mtx(chan));
                scsipi_user_done(xs);
                SC_DEBUG(periph, SCSIPI_DB3, ("returned from user done()\n "));
                return 0;
        }

        switch (xs->error) {
        case XS_NOERROR:
                error = 0;
                break;

        case XS_SENSE:
        case XS_SHORTSENSE:
                error = (*chan->chan_bustype->bustype_interpret_sense)(xs);
                break;

        case XS_RESOURCE_SHORTAGE:
                /*
                 * XXX Should freeze channel's queue.
                 */
                scsipi_printaddr(periph);
                printf("adapter resource shortage\n");
                /* FALLTHROUGH */

        case XS_BUSY:
                if (xs->error == XS_BUSY && xs->status == SCSI_QUEUE_FULL) {
                        struct scsipi_max_openings mo;

                        /*
                         * We set the openings to active - 1, assuming that
                         * the command that got us here is the first one that
                         * can't fit into the device's queue.  If that's not
                         * the case, I guess we'll find out soon enough.
                         */
                        mo.mo_target = periph->periph_target;
                        mo.mo_lun = periph->periph_lun;
                        if (periph->periph_active < periph->periph_openings)
                                mo.mo_openings = periph->periph_active - 1;
                        else
                                mo.mo_openings = periph->periph_openings - 1;
#ifdef DIAGNOSTIC
                        if (mo.mo_openings < 0) {
                                scsipi_printaddr(periph);
                                printf("QUEUE FULL resulted in < 0 openings\n");
                                panic("scsipi_done");
                        }
#endif
                        if (mo.mo_openings == 0) {
                                scsipi_printaddr(periph);
                                printf("QUEUE FULL resulted in 0 openings\n");
                                mo.mo_openings = 1;
                        }
                        scsipi_async_event(chan, ASYNC_EVENT_MAX_OPENINGS, &mo);
                        error = ERESTART;
                } else if (xs->xs_retries != 0) {
                        xs->xs_retries--;
                        /*
                         * Wait one second, and try again.
                         */
                        mutex_enter(chan_mtx(chan));
                        if ((xs->xs_control & XS_CTL_POLL) ||
                            (chan->chan_flags & SCSIPI_CHAN_TACTIVE) == 0) {
                                /* XXX: quite extreme */
                                kpause("xsbusy", false, hz, chan_mtx(chan));
                        } else if (!callout_pending(&periph->periph_callout)) {
                                scsipi_periph_freeze_locked(periph, 1);
                                callout_reset(&periph->periph_callout,
                                    hz, scsipi_periph_timed_thaw, periph);
                        }
                        mutex_exit(chan_mtx(chan));
                        error = ERESTART;
                } else
                        error = EBUSY;
                break;

        case XS_REQUEUE:
                error = ERESTART;
                break;

        case XS_SELTIMEOUT:
        case XS_TIMEOUT:
                /*
                 * If the device hasn't gone away, honor retry counts.
                 *
                 * Note that if we're in the middle of probing it,
                 * it won't be found because it isn't here yet so
                 * we won't honor the retry count in that case.
                 */
                if (scsipi_lookup_periph(chan, periph->periph_target,
                    periph->periph_lun) && xs->xs_retries != 0) {
                        xs->xs_retries--;
                        error = ERESTART;
                } else
                        error = EIO;
                break;

        case XS_RESET:
                if (xs->xs_control & XS_CTL_REQSENSE) {
                        /*
                         * request sense interrupted by reset: signal it
                         * with EINTR return code.
                         */
                        error = EINTR;
                } else {
                        if (xs->xs_retries != 0) {
                                xs->xs_retries--;
                                error = ERESTART;
                        } else
                                error = EIO;
                }
                break;

        case XS_DRIVER_STUFFUP:
                scsipi_printaddr(periph);
                printf("generic HBA error\n");
                error = EIO;
                break;
        default:
                scsipi_printaddr(periph);
                printf("invalid return code from adapter: %d\n", xs->error);
                error = EIO;
                break;
        }

        mutex_enter(chan_mtx(chan));
        if (error == ERESTART) {
                SDT_PROBE1(scsi, base, xfer, restart,  xs);
                /*
                 * If we get here, the periph has been thawed and frozen
                 * again if we had to issue recovery commands.  Alternatively,
                 * it may have been frozen again and in a timed thaw.  In
                 * any case, we thaw the periph once we re-enqueue the
                 * command.  Once the periph is fully thawed, it will begin
                 * operation again.
                 */
                xs->error = XS_NOERROR;
                xs->status = SCSI_OK;
                xs->xs_status &= ~XS_STS_DONE;
                xs->xs_requeuecnt++;
                error = scsipi_enqueue(xs);
                if (error == 0) {
                        scsipi_periph_thaw_locked(periph, 1);
                        mutex_exit(chan_mtx(chan));
                        return ERESTART;
                }
        }

        /*
         * scsipi_done() freezes the queue if not XS_NOERROR.
         * Thaw it here.
         */
        if (xs->error != XS_NOERROR)
                scsipi_periph_thaw_locked(periph, 1);
        mutex_exit(chan_mtx(chan));

        if (periph->periph_switch->psw_done)
                periph->periph_switch->psw_done(xs, error);

        mutex_enter(chan_mtx(chan));
        if (xs->xs_control & XS_CTL_ASYNC)
                scsipi_put_xs(xs);
        mutex_exit(chan_mtx(chan));

        return error;
}

/*
 * Issue a request sense for the given scsipi_xfer. Called when the xfer
 * returns with a CHECK_CONDITION status. Must be called in valid thread
 * context.
 */

static void
scsipi_request_sense(struct scsipi_xfer *xs)
{
        struct scsipi_periph *periph = xs->xs_periph;
        int flags, error;
        struct scsi_request_sense cmd;

        periph->periph_flags |= PERIPH_SENSE;

        /* if command was polling, request sense will too */
        flags = xs->xs_control & XS_CTL_POLL;
        /* Polling commands can't sleep */
        if (flags)
                flags |= XS_CTL_NOSLEEP;

        flags |= XS_CTL_REQSENSE | XS_CTL_URGENT | XS_CTL_DATA_IN |
            XS_CTL_THAW_PERIPH | XS_CTL_FREEZE_PERIPH;

        memset(&cmd, 0, sizeof(cmd));
        cmd.opcode = SCSI_REQUEST_SENSE;
        cmd.length = sizeof(struct scsi_sense_data);

        error = scsipi_command(periph, (void *)&cmd, sizeof(cmd),
            (void *)&xs->sense.scsi_sense, sizeof(struct scsi_sense_data),
            0, 1000, NULL, flags);
        periph->periph_flags &= ~PERIPH_SENSE;
        periph->periph_xscheck = NULL;
        switch (error) {
        case 0:
                /* we have a valid sense */
                xs->error = XS_SENSE;
                return;
        case EINTR:
                /* REQUEST_SENSE interrupted by bus reset. */
                xs->error = XS_RESET;
                return;
        case EIO:
                 /* request sense couldn't be performed */
                /*
                 * XXX this isn't quite right but we don't have anything
                 * better for now
                 */
                xs->error = XS_DRIVER_STUFFUP;
                return;
        default:
                 /* Notify that request sense failed. */
                xs->error = XS_DRIVER_STUFFUP;
                scsipi_printaddr(periph);
                printf("request sense failed with error %d\n", error);
                return;
        }
}

/*
 * scsipi_enqueue:
 *
 *      Enqueue an xfer on a channel.
 */
static int
scsipi_enqueue(struct scsipi_xfer *xs)
{
        struct scsipi_channel *chan = xs->xs_periph->periph_channel;
        struct scsipi_xfer *qxs;

        SDT_PROBE1(scsi, base, xfer, enqueue,  xs);

        /*
         * If the xfer is to be polled, and there are already jobs on
         * the queue, we can't proceed.
         */
        KASSERT(mutex_owned(chan_mtx(chan)));
        if ((xs->xs_control & XS_CTL_POLL) != 0 &&
            TAILQ_FIRST(&chan->chan_queue) != NULL) {
                xs->error = XS_DRIVER_STUFFUP;
                return EAGAIN;
        }

        /*
         * If we have an URGENT xfer, it's an error recovery command
         * and it should just go on the head of the channel's queue.
         */
        if (xs->xs_control & XS_CTL_URGENT) {
                TAILQ_INSERT_HEAD(&chan->chan_queue, xs, channel_q);
                goto out;
        }

        /*
         * If this xfer has already been on the queue before, we
         * need to reinsert it in the correct order.  That order is:
         *
         *      Immediately before the first xfer for this periph
         *      with a requeuecnt less than xs->xs_requeuecnt.
         *
         * Failing that, at the end of the queue.  (We'll end up
         * there naturally.)
         */
        if (xs->xs_requeuecnt != 0) {
                for (qxs = TAILQ_FIRST(&chan->chan_queue); qxs != NULL;
                     qxs = TAILQ_NEXT(qxs, channel_q)) {
                        if (qxs->xs_periph == xs->xs_periph &&
                            qxs->xs_requeuecnt < xs->xs_requeuecnt)
                                break;
                }
                if (qxs != NULL) {
                        TAILQ_INSERT_AFTER(&chan->chan_queue, qxs, xs,
                            channel_q);
                        goto out;
                }
        }
        TAILQ_INSERT_TAIL(&chan->chan_queue, xs, channel_q);
 out:
        if (xs->xs_control & XS_CTL_THAW_PERIPH)
                scsipi_periph_thaw_locked(xs->xs_periph, 1);
        return 0;
}

/*
 * scsipi_run_queue:
 *
 *      Start as many xfers as possible running on the channel.
 */
static void
scsipi_run_queue(struct scsipi_channel *chan)
{
        struct scsipi_xfer *xs;
        struct scsipi_periph *periph;

        SDT_PROBE1(scsi, base, queue, batch__start,  chan);
        for (;;) {
                mutex_enter(chan_mtx(chan));

                /*
                 * If the channel is frozen, we can't do any work right
                 * now.
                 */
                if (chan->chan_qfreeze != 0) {
                        mutex_exit(chan_mtx(chan));
                        break;
                }

                /*
                 * Look for work to do, and make sure we can do it.
                 */
                for (xs = TAILQ_FIRST(&chan->chan_queue); xs != NULL;
                     xs = TAILQ_NEXT(xs, channel_q)) {
                        periph = xs->xs_periph;

                        if ((periph->periph_sent >= periph->periph_openings) ||
                            periph->periph_qfreeze != 0 ||
                            (periph->periph_flags & PERIPH_UNTAG) != 0)
                                continue;

                        if ((periph->periph_flags &
                            (PERIPH_RECOVERING | PERIPH_SENSE)) != 0 &&
                            (xs->xs_control & XS_CTL_URGENT) == 0)
                                continue;

                        /*
                         * We can issue this xfer!
                         */
                        goto got_one;
                }

                /*
                 * Can't find any work to do right now.
                 */
                mutex_exit(chan_mtx(chan));
                break;

 got_one:
                /*
                 * Have an xfer to run.  Allocate a resource from
                 * the adapter to run it.  If we can't allocate that
                 * resource, we don't dequeue the xfer.
                 */
                if (scsipi_get_resource(chan) == 0) {
                        /*
                         * Adapter is out of resources.  If the adapter
                         * supports it, attempt to grow them.
                         */
                        if (scsipi_grow_resources(chan) == 0) {
                                /*
                                 * Wasn't able to grow resources,
                                 * nothing more we can do.
                                 */
                                if (xs->xs_control & XS_CTL_POLL) {
                                        scsipi_printaddr(xs->xs_periph);
                                        printf("polling command but no "
                                            "adapter resources");
                                        /* We'll panic shortly... */
                                }
                                mutex_exit(chan_mtx(chan));

                                /*
                                 * XXX: We should be able to note that
                                 * XXX: that resources are needed here!
                                 */
                                break;
                        }
                        /*
                         * scsipi_grow_resources() allocated the resource
                         * for us.
                         */
                }

                /*
                 * We have a resource to run this xfer, do it!
                 */
                TAILQ_REMOVE(&chan->chan_queue, xs, channel_q);

                /*
                 * If the command is to be tagged, allocate a tag ID
                 * for it.
                 */
                if (XS_CTL_TAGTYPE(xs) != 0)
                        scsipi_get_tag(xs);
                else
                        periph->periph_flags |= PERIPH_UNTAG;
                periph->periph_sent++;
                mutex_exit(chan_mtx(chan));

                SDT_PROBE2(scsi, base, queue, run,  chan, xs);
                scsipi_adapter_request(chan, ADAPTER_REQ_RUN_XFER, xs);
        }
        SDT_PROBE1(scsi, base, queue, batch__done,  chan);
}

/*
 * scsipi_execute_xs:
 *
 *      Begin execution of an xfer, waiting for it to complete, if necessary.
 */
int
scsipi_execute_xs(struct scsipi_xfer *xs)
{
        struct scsipi_periph *periph = xs->xs_periph;
        struct scsipi_channel *chan = periph->periph_channel;
        int oasync, async, poll, error;

        KASSERT(!cold);

        scsipi_update_timeouts(xs);
        
        (chan->chan_bustype->bustype_cmd)(xs);

        xs->xs_status &= ~XS_STS_DONE;
        xs->error = XS_NOERROR;
        xs->resid = xs->datalen;
        xs->status = SCSI_OK;
        SDT_PROBE1(scsi, base, xfer, execute,  xs);

#ifdef SCSIPI_DEBUG
        if (xs->xs_periph->periph_dbflags & SCSIPI_DB3) {
                printf("scsipi_execute_xs: ");
                show_scsipi_xs(xs);
                printf("\n");
        }
#endif

        /*
         * Deal with command tagging:
         *
         *      - If the device's current operating mode doesn't
         *        include tagged queueing, clear the tag mask.
         *
         *      - If the device's current operating mode *does*
         *        include tagged queueing, set the tag_type in
         *        the xfer to the appropriate byte for the tag
         *        message.
         */
        if ((PERIPH_XFER_MODE(periph) & PERIPH_CAP_TQING) == 0 ||
                (xs->xs_control & XS_CTL_REQSENSE)) {
                xs->xs_control &= ~XS_CTL_TAGMASK;
                xs->xs_tag_type = 0;
        } else {
                /*
                 * If the request doesn't specify a tag, give Head
                 * tags to URGENT operations and Simple tags to
                 * everything else.
                 */
                if (XS_CTL_TAGTYPE(xs) == 0) {
                        if (xs->xs_control & XS_CTL_URGENT)
                                xs->xs_control |= XS_CTL_HEAD_TAG;
                        else
                                xs->xs_control |= XS_CTL_SIMPLE_TAG;
                }

                switch (XS_CTL_TAGTYPE(xs)) {
                case XS_CTL_ORDERED_TAG:
                        xs->xs_tag_type = MSG_ORDERED_Q_TAG;
                        break;

                case XS_CTL_SIMPLE_TAG:
                        xs->xs_tag_type = MSG_SIMPLE_Q_TAG;
                        break;

                case XS_CTL_HEAD_TAG:
                        xs->xs_tag_type = MSG_HEAD_OF_Q_TAG;
                        break;

                default:
                        scsipi_printaddr(periph);
                        printf("invalid tag mask 0x%08x\n",
                            XS_CTL_TAGTYPE(xs));
                        panic("scsipi_execute_xs");
                }
        }

        /* If the adapter wants us to poll, poll. */
        if (chan->chan_adapter->adapt_flags & SCSIPI_ADAPT_POLL_ONLY)
                xs->xs_control |= XS_CTL_POLL;

        /*
         * If we don't yet have a completion thread, or we are to poll for
         * completion, clear the ASYNC flag.
         */
        oasync =  (xs->xs_control & XS_CTL_ASYNC);
        if (chan->chan_thread == NULL || (xs->xs_control & XS_CTL_POLL) != 0)
                xs->xs_control &= ~XS_CTL_ASYNC;

        async = (xs->xs_control & XS_CTL_ASYNC);
        poll = (xs->xs_control & XS_CTL_POLL);

#ifdef DIAGNOSTIC
        if (oasync != 0 && xs->bp == NULL)
                panic("scsipi_execute_xs: XS_CTL_ASYNC but no buf");
#endif

        /*
         * Enqueue the transfer.  If we're not polling for completion, this
         * should ALWAYS return `no error'.
         */
        error = scsipi_enqueue(xs);
        if (error) {
                if (poll == 0) {
                        scsipi_printaddr(periph);
                        printf("not polling, but enqueue failed with %d\n",
                            error);
                        panic("scsipi_execute_xs");
                }

                scsipi_printaddr(periph);
                printf("should have flushed queue?\n");
                goto free_xs;
        }

        mutex_exit(chan_mtx(chan));
 restarted:
        scsipi_run_queue(chan);
        mutex_enter(chan_mtx(chan));

        /*
         * The xfer is enqueued, and possibly running.  If it's to be
         * completed asynchronously, just return now.
         */
        if (async)
                return 0;

        /*
         * Not an asynchronous command; wait for it to complete.
         */
        while ((xs->xs_status & XS_STS_DONE) == 0) {
                if (poll) {
                        scsipi_printaddr(periph);
                        printf("polling command not done\n");
                        panic("scsipi_execute_xs");
                }
                cv_wait(xs_cv(xs), chan_mtx(chan));
        }

        /*
         * Command is complete.  scsipi_done() has awakened us to perform
         * the error handling.
         */
        mutex_exit(chan_mtx(chan));
        error = scsipi_complete(xs);
        if (error == ERESTART)
                goto restarted;

        /*
         * If it was meant to run async and we cleared async ourselves,
         * don't return an error here. It has already been handled
         */
        if (oasync)
                error = 0;
        /*
         * Command completed successfully or fatal error occurred.  Fall
         * into....
         */
        mutex_enter(chan_mtx(chan));
 free_xs:
        scsipi_put_xs(xs);
        mutex_exit(chan_mtx(chan));

        /*
         * Kick the queue, keep it running in case it stopped for some
         * reason.
         */
        scsipi_run_queue(chan);

        mutex_enter(chan_mtx(chan));
        return error;
}

/*
 * scsipi_completion_thread:
 *
 *      This is the completion thread.  We wait for errors on
 *      asynchronous xfers, and perform the error handling
 *      function, restarting the command, if necessary.
 */
static void
scsipi_completion_thread(void *arg)
{
        struct scsipi_channel *chan = arg;
        struct scsipi_xfer *xs;

        if (chan->chan_init_cb)
                (*chan->chan_init_cb)(chan, chan->chan_init_cb_arg);

        mutex_enter(chan_mtx(chan));
        chan->chan_flags |= SCSIPI_CHAN_TACTIVE;
        for (;;) {
                xs = TAILQ_FIRST(&chan->chan_complete);
                if (xs == NULL && chan->chan_tflags == 0) {
                        /* nothing to do; wait */
                        cv_wait(chan_cv_complete(chan), chan_mtx(chan));
                        continue;
                }
                if (chan->chan_tflags & SCSIPI_CHANT_CALLBACK) {
                        /* call chan_callback from thread context */
                        chan->chan_tflags &= ~SCSIPI_CHANT_CALLBACK;
                        chan->chan_callback(chan, chan->chan_callback_arg);
                        continue;
                }
                if (chan->chan_tflags & SCSIPI_CHANT_GROWRES) {
                        /* attempt to get more openings for this channel */
                        chan->chan_tflags &= ~SCSIPI_CHANT_GROWRES;
                        mutex_exit(chan_mtx(chan));
                        scsipi_adapter_request(chan,
                            ADAPTER_REQ_GROW_RESOURCES, NULL);
                        scsipi_channel_thaw(chan, 1);
                        if (chan->chan_tflags & SCSIPI_CHANT_GROWRES)
                                kpause("scsizzz", FALSE, hz/10, NULL);
                        mutex_enter(chan_mtx(chan));
                        continue;
                }
                if (chan->chan_tflags & SCSIPI_CHANT_KICK) {
                        /* explicitly run the queues for this channel */
                        chan->chan_tflags &= ~SCSIPI_CHANT_KICK;
                        mutex_exit(chan_mtx(chan));
                        scsipi_run_queue(chan);
                        mutex_enter(chan_mtx(chan));
                        continue;
                }
                if (chan->chan_tflags & SCSIPI_CHANT_SHUTDOWN) {
                        break;
                }
                if (xs) {
                        TAILQ_REMOVE(&chan->chan_complete, xs, channel_q);
                        mutex_exit(chan_mtx(chan));

                        /*
                         * Have an xfer with an error; process it.
                         */
                        (void) scsipi_complete(xs);

                        /*
                         * Kick the queue; keep it running if it was stopped
                         * for some reason.
                         */
                        scsipi_run_queue(chan);
                        mutex_enter(chan_mtx(chan));
                }
        }

        chan->chan_thread = NULL;

        /* In case parent is waiting for us to exit. */
        cv_broadcast(chan_cv_thread(chan));
        mutex_exit(chan_mtx(chan));

        kthread_exit(0);
}
/*
 * scsipi_thread_call_callback:
 *
 *      request to call a callback from the completion thread
 */
int
scsipi_thread_call_callback(struct scsipi_channel *chan,
    void (*callback)(struct scsipi_channel *, void *), void *arg)
{

        mutex_enter(chan_mtx(chan));
        if ((chan->chan_flags & SCSIPI_CHAN_TACTIVE) == 0) {
                /* kernel thread doesn't exist yet */
                mutex_exit(chan_mtx(chan));
                return ESRCH;
        }
        if (chan->chan_tflags & SCSIPI_CHANT_CALLBACK) {
                mutex_exit(chan_mtx(chan));
                return EBUSY;
        }
        scsipi_channel_freeze(chan, 1);
        chan->chan_callback = callback;
        chan->chan_callback_arg = arg;
        chan->chan_tflags |= SCSIPI_CHANT_CALLBACK;
        cv_broadcast(chan_cv_complete(chan));
        mutex_exit(chan_mtx(chan));
        return 0;
}

/*
 * scsipi_async_event:
 *
 *      Handle an asynchronous event from an adapter.
 */
void
scsipi_async_event(struct scsipi_channel *chan, scsipi_async_event_t event,
    void *arg)
{
        bool lock = chan_running(chan) > 0;

        if (lock)
                mutex_enter(chan_mtx(chan));
        switch (event) {
        case ASYNC_EVENT_MAX_OPENINGS:
                scsipi_async_event_max_openings(chan,
                    (struct scsipi_max_openings *)arg);
                break;

        case ASYNC_EVENT_XFER_MODE:
                if (chan->chan_bustype->bustype_async_event_xfer_mode) {
                        chan->chan_bustype->bustype_async_event_xfer_mode(
                            chan, arg);
                }
                break;
        case ASYNC_EVENT_RESET:
                scsipi_async_event_channel_reset(chan);
                break;
        }
        if (lock)
                mutex_exit(chan_mtx(chan));
}

/*
 * scsipi_async_event_max_openings:
 *
 *      Update the maximum number of outstanding commands a
 *      device may have.
 */
static void
scsipi_async_event_max_openings(struct scsipi_channel *chan,
    struct scsipi_max_openings *mo)
{
        struct scsipi_periph *periph;
        int minlun, maxlun;

        if (mo->mo_lun == -1) {
                /*
                 * Wildcarded; apply it to all LUNs.
                 */
                minlun = 0;
                maxlun = chan->chan_nluns - 1;
        } else
                minlun = maxlun = mo->mo_lun;

        /* XXX This could really suck with a large LUN space. */
        for (; minlun <= maxlun; minlun++) {
                periph = scsipi_lookup_periph_locked(chan, mo->mo_target, minlun);
                if (periph == NULL)
                        continue;

                if (mo->mo_openings < periph->periph_openings)
                        periph->periph_openings = mo->mo_openings;
                else if (mo->mo_openings > periph->periph_openings &&
                    (periph->periph_flags & PERIPH_GROW_OPENINGS) != 0)
                        periph->periph_openings = mo->mo_openings;
        }
}

/*
 * scsipi_set_xfer_mode:
 *
 *      Set the xfer mode for the specified I_T Nexus.
 */
void
scsipi_set_xfer_mode(struct scsipi_channel *chan, int target, int immed)
{
        struct scsipi_xfer_mode xm;
        struct scsipi_periph *itperiph;
        int lun;

        /*
         * Go to the minimal xfer mode.
         */
        xm.xm_target = target;
        xm.xm_mode = 0;
        xm.xm_period = 0;                       /* ignored */
        xm.xm_offset = 0;                       /* ignored */

        /*
         * Find the first LUN we know about on this I_T Nexus.
         */
        for (itperiph = NULL, lun = 0; lun < chan->chan_nluns; lun++) {
                itperiph = scsipi_lookup_periph(chan, target, lun);
                if (itperiph != NULL)
                        break;
        }
        if (itperiph != NULL) {
                xm.xm_mode = itperiph->periph_cap;
                /*
                 * Now issue the request to the adapter.
                 */
                scsipi_adapter_request(chan, ADAPTER_REQ_SET_XFER_MODE, &xm);
                /*
                 * If we want this to happen immediately, issue a dummy
                 * command, since most adapters can't really negotiate unless
                 * they're executing a job.
                 */
                if (immed != 0) {
                        (void) scsipi_test_unit_ready(itperiph,
                            XS_CTL_DISCOVERY | XS_CTL_IGNORE_ILLEGAL_REQUEST |
                            XS_CTL_IGNORE_NOT_READY |
                            XS_CTL_IGNORE_MEDIA_CHANGE);
                }
        }
}

/*
 * scsipi_channel_reset:
 *
 *      handle scsi bus reset
 * called with channel lock held
 */
static void
scsipi_async_event_channel_reset(struct scsipi_channel *chan)
{
        struct scsipi_xfer *xs, *xs_next;
        struct scsipi_periph *periph;
        int target, lun;

        /*
         * Channel has been reset. Also mark as reset pending REQUEST_SENSE
         * commands; as the sense is not available any more.
         * can't call scsipi_done() from here, as the command has not been
         * sent to the adapter yet (this would corrupt accounting).
         */

        for (xs = TAILQ_FIRST(&chan->chan_queue); xs != NULL; xs = xs_next) {
                xs_next = TAILQ_NEXT(xs, channel_q);
                if (xs->xs_control & XS_CTL_REQSENSE) {
                        TAILQ_REMOVE(&chan->chan_queue, xs, channel_q);
                        xs->error = XS_RESET;
                        if ((xs->xs_control & XS_CTL_ASYNC) != 0)
                                TAILQ_INSERT_TAIL(&chan->chan_complete, xs,
                                    channel_q);
                }
        }
        cv_broadcast(chan_cv_complete(chan));
        /* Catch xs with pending sense which may not have a REQSENSE xs yet */
        for (target = 0; target < chan->chan_ntargets; target++) {
                if (target == chan->chan_id)
                        continue;
                for (lun = 0; lun <  chan->chan_nluns; lun++) {
                        periph = scsipi_lookup_periph_locked(chan, target, lun);
                        if (periph) {
                                xs = periph->periph_xscheck;
                                if (xs)
                                        xs->error = XS_RESET;
                        }
                }
        }
}

/*
 * scsipi_target_detach:
 *
 *      detach all periph associated with a I_T
 *      must be called from valid thread context
 */
int
scsipi_target_detach(struct scsipi_channel *chan, int target, int lun,
    int flags)
{
        struct scsipi_periph *periph;
        device_t tdev;
        int ctarget, mintarget, maxtarget;
        int clun, minlun, maxlun;
        int error = 0;

        if (target == -1) {
                mintarget = 0;
                maxtarget = chan->chan_ntargets;
        } else {
                if (target == chan->chan_id)
                        return EINVAL;
                if (target < 0 || target >= chan->chan_ntargets)
                        return EINVAL;
                mintarget = target;
                maxtarget = target + 1;
        }

        if (lun == -1) {
                minlun = 0;
                maxlun = chan->chan_nluns;
        } else {
                if (lun < 0 || lun >= chan->chan_nluns)
                        return EINVAL;
                minlun = lun;
                maxlun = lun + 1;
        }

        /* for config_detach */
        KERNEL_LOCK(1, curlwp);

        mutex_enter(chan_mtx(chan));
        for (ctarget = mintarget; ctarget < maxtarget; ctarget++) {
                if (ctarget == chan->chan_id)
                        continue;

                for (clun = minlun; clun < maxlun; clun++) {
                        periph = scsipi_lookup_periph_locked(chan, ctarget, clun);
                        if (periph == NULL)
                                continue;
                        tdev = periph->periph_dev;
                        mutex_exit(chan_mtx(chan));
                        error = config_detach(tdev, flags);
                        if (error)
                                goto out;
                        mutex_enter(chan_mtx(chan));
                        KASSERT(scsipi_lookup_periph_locked(chan, ctarget, clun) == NULL);
                }
        }
        mutex_exit(chan_mtx(chan));

out:
        KERNEL_UNLOCK_ONE(curlwp);

        return error;
}

/*
 * scsipi_adapter_addref:
 *
 *      Add a reference to the adapter pointed to by the provided
 *      link, enabling the adapter if necessary.
 */
int
scsipi_adapter_addref(struct scsipi_adapter *adapt)
{
        int error = 0;

        if (atomic_inc_uint_nv(&adapt->adapt_refcnt) == 1
            && adapt->adapt_enable != NULL) {
                scsipi_adapter_lock(adapt);
                error = scsipi_adapter_enable(adapt, 1);
                scsipi_adapter_unlock(adapt);
                if (error)
                        atomic_dec_uint(&adapt->adapt_refcnt);
        }
        return error;
}

/*
 * scsipi_adapter_delref:
 *
 *      Delete a reference to the adapter pointed to by the provided
 *      link, disabling the adapter if possible.
 */
void
scsipi_adapter_delref(struct scsipi_adapter *adapt)
{

        membar_release();
        if (atomic_dec_uint_nv(&adapt->adapt_refcnt) == 0
            && adapt->adapt_enable != NULL) {
                membar_acquire();
                scsipi_adapter_lock(adapt);
                (void) scsipi_adapter_enable(adapt, 0);
                scsipi_adapter_unlock(adapt);
        }
}

static struct scsipi_syncparam {
        int     ss_factor;
        int     ss_period;      /* ns * 100 */
} scsipi_syncparams[] = {
        { 0x08,          625 }, /* FAST-160 (Ultra320) */
        { 0x09,         1250 }, /* FAST-80 (Ultra160) */
        { 0x0a,         2500 }, /* FAST-40 40MHz (Ultra2) */
        { 0x0b,         3030 }, /* FAST-40 33MHz (Ultra2) */
        { 0x0c,         5000 }, /* FAST-20 (Ultra) */
};
static const int scsipi_nsyncparams =
    sizeof(scsipi_syncparams) / sizeof(scsipi_syncparams[0]);

int
scsipi_sync_period_to_factor(int period /* ns * 100 */)
{
        int i;

        for (i = 0; i < scsipi_nsyncparams; i++) {
                if (period <= scsipi_syncparams[i].ss_period)
                        return scsipi_syncparams[i].ss_factor;
        }

        return (period / 100) / 4;
}

int
scsipi_sync_factor_to_period(int factor)
{
        int i;

        for (i = 0; i < scsipi_nsyncparams; i++) {
                if (factor == scsipi_syncparams[i].ss_factor)
                        return scsipi_syncparams[i].ss_period;
        }

        return (factor * 4) * 100;
}

int
scsipi_sync_factor_to_freq(int factor)
{
        int i;

        for (i = 0; i < scsipi_nsyncparams; i++) {
                if (factor == scsipi_syncparams[i].ss_factor)
                        return 100000000 / scsipi_syncparams[i].ss_period;
        }

        return 10000000 / ((factor * 4) * 10);
}

static inline void
scsipi_adapter_lock(struct scsipi_adapter *adapt)
{

        if ((adapt->adapt_flags & SCSIPI_ADAPT_MPSAFE) == 0)
                KERNEL_LOCK(1, NULL);
}

static inline void
scsipi_adapter_unlock(struct scsipi_adapter *adapt)
{

        if ((adapt->adapt_flags & SCSIPI_ADAPT_MPSAFE) == 0)
                KERNEL_UNLOCK_ONE(NULL);
}

void
scsipi_adapter_minphys(struct scsipi_channel *chan, struct buf *bp)
{
        struct scsipi_adapter *adapt = chan->chan_adapter;

        scsipi_adapter_lock(adapt);
        (adapt->adapt_minphys)(bp);
        scsipi_adapter_unlock(chan->chan_adapter);
}

void
scsipi_adapter_request(struct scsipi_channel *chan, 
        scsipi_adapter_req_t req, void *arg)

{
        struct scsipi_adapter *adapt = chan->chan_adapter;

        scsipi_adapter_lock(adapt);
        SDT_PROBE3(scsi, base, adapter, request__start,  chan, req, arg);
        (adapt->adapt_request)(chan, req, arg);
        SDT_PROBE3(scsi, base, adapter, request__done,  chan, req, arg);
        scsipi_adapter_unlock(adapt);
}

int
scsipi_adapter_ioctl(struct scsipi_channel *chan, u_long cmd,
        void *data, int flag, struct proc *p)
{
        struct scsipi_adapter *adapt = chan->chan_adapter;
        int error;

        if (adapt->adapt_ioctl == NULL)
                return ENOTTY;

        scsipi_adapter_lock(adapt);
        error = (adapt->adapt_ioctl)(chan, cmd, data, flag, p);
        scsipi_adapter_unlock(adapt);
        return error;
}

int
scsipi_adapter_enable(struct scsipi_adapter *adapt, int enable)
{
        int error;

        scsipi_adapter_lock(adapt);
        error = (adapt->adapt_enable)(adapt->adapt_dev, enable);
        scsipi_adapter_unlock(adapt);
        return error;
}

#ifdef SCSIPI_DEBUG
/*
 * Given a scsipi_xfer, dump the request, in all its glory
 */
void
show_scsipi_xs(struct scsipi_xfer *xs)
{

        printf("xs(%p): ", xs);
        printf("xs_control(0x%08x)", xs->xs_control);
        printf("xs_status(0x%08x)", xs->xs_status);
        printf("periph(%p)", xs->xs_periph);
        printf("retr(0x%x)", xs->xs_retries);
        printf("timo(0x%x)", xs->timeout);
        printf("cmd(%p)", xs->cmd);
        printf("len(0x%x)", xs->cmdlen);
        printf("data(%p)", xs->data);
        printf("len(0x%x)", xs->datalen);
        printf("res(0x%x)", xs->resid);
        printf("err(0x%x)", xs->error);
        printf("bp(%p)", xs->bp);
        show_scsipi_cmd(xs);
}

void
show_scsipi_cmd(struct scsipi_xfer *xs)
{
        u_char *b = (u_char *) xs->cmd;
        int i = 0;

        scsipi_printaddr(xs->xs_periph);
        printf(" command: ");

        if ((xs->xs_control & XS_CTL_RESET) == 0) {
                while (i < xs->cmdlen) {
                        if (i)
                                printf(",");
                        printf("0x%x", b[i++]);
                }
                printf("-[%d bytes]\n", xs->datalen);
                if (xs->datalen)
                        show_mem(xs->data, uimin(64, xs->datalen));
        } else
                printf("-RESET-\n");
}

void
show_mem(u_char *address, int num)
{
        int x;

        printf("------------------------------");
        for (x = 0; x < num; x++) {
                if ((x % 16) == 0)
                        printf("\n%03d: ", x);
                printf("%02x ", *address++);
        }
        printf("\n------------------------------\n");
}
#endif /* SCSIPI_DEBUG */