/*-
 * SPDX-License-Identifier: BSD-2-Clause
 *
 * Copyright (c) 2009 Alexander Motin <mav@FreeBSD.org>
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions
 * are met:
 * 1. Redistributions of source code must retain the above copyright
 *    notice, this list of conditions and the following disclaimer,
 *    without modification, immediately at the beginning of the file.
 * 2. Redistributions in binary form must reproduce the above copyright
 *    notice, this list of conditions and the following disclaimer in the
 *    documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

#include <sys/param.h>
#include <sys/bus.h>
#include <sys/endian.h>
#include <sys/systm.h>
#include <sys/types.h>
#include <sys/malloc.h>
#include <sys/kernel.h>
#include <sys/time.h>
#include <sys/conf.h>
#include <sys/fcntl.h>
#include <sys/sbuf.h>
#include <sys/stdarg.h>

#include <sys/eventhandler.h>
#include <sys/lock.h>
#include <sys/mutex.h>
#include <sys/sysctl.h>

#include <cam/cam.h>
#include <cam/cam_ccb.h>
#include <cam/cam_queue.h>
#include <cam/cam_periph.h>
#include <cam/cam_sim.h>
#include <cam/cam_xpt.h>
#include <cam/cam_xpt_sim.h>
#include <cam/cam_xpt_periph.h>
#include <cam/cam_xpt_internal.h>
#include <cam/cam_debug.h>

#include <cam/scsi/scsi_all.h>
#include <cam/scsi/scsi_message.h>
#include <cam/ata/ata_all.h>

struct ata_quirk_entry {
        struct scsi_inquiry_pattern inq_pat;
        uint8_t quirks;
#define CAM_QUIRK_MAXTAGS       0x01
        u_int mintags;
        u_int maxtags;
};

static periph_init_t aprobe_periph_init;

static struct periph_driver aprobe_driver =
{
        aprobe_periph_init, "aprobe",
        TAILQ_HEAD_INITIALIZER(aprobe_driver.units), /* generation */ 0,
        CAM_PERIPH_DRV_EARLY
};

PERIPHDRIVER_DECLARE(aprobe, aprobe_driver);

typedef enum {
        PROBE_RESET,
        PROBE_IDENTIFY,
        PROBE_SPINUP,
        PROBE_SETMODE,
        PROBE_SETPM,
        PROBE_SETAPST,
        PROBE_SETDMAAA,
        PROBE_SETAN,
        PROBE_SET_MULTI,
        PROBE_INQUIRY,
        PROBE_FULL_INQUIRY,
        PROBE_PM_PID,
        PROBE_PM_PRV,
        PROBE_IDENTIFY_SES,
        PROBE_IDENTIFY_SAFTE,
        PROBE_DONE,
        PROBE_INVALID
} aprobe_action;

static char *probe_action_text[] = {
        "PROBE_RESET",
        "PROBE_IDENTIFY",
        "PROBE_SPINUP",
        "PROBE_SETMODE",
        "PROBE_SETPM",
        "PROBE_SETAPST",
        "PROBE_SETDMAAA",
        "PROBE_SETAN",
        "PROBE_SET_MULTI",
        "PROBE_INQUIRY",
        "PROBE_FULL_INQUIRY",
        "PROBE_PM_PID",
        "PROBE_PM_PRV",
        "PROBE_IDENTIFY_SES",
        "PROBE_IDENTIFY_SAFTE",
        "PROBE_DONE",
        "PROBE_INVALID"
};

#define PROBE_SET_ACTION(softc, newaction)      \
do {                                                                    \
        char **text;                                                    \
        text = probe_action_text;                                       \
        CAM_DEBUG((softc)->periph->path, CAM_DEBUG_PROBE,               \
            ("Probe %s to %s\n", text[(softc)->action],                 \
            text[(newaction)]));                                        \
        (softc)->action = (newaction);                                  \
} while(0)

typedef enum {
        PROBE_NO_ANNOUNCE       = 0x04
} aprobe_flags;

typedef struct {
        TAILQ_HEAD(, ccb_hdr) request_ccbs;
        struct ata_params       ident_data;
        aprobe_action   action;
        aprobe_flags    flags;
        uint32_t        pm_pid;
        uint32_t        pm_prv;
        int             restart;
        int             spinup;
        int             faults;
        u_int           caps;
        struct cam_periph *periph;
} aprobe_softc;

static struct ata_quirk_entry ata_quirk_table[] =
{
        {
                /* Default tagged queuing parameters for all devices */
                {
                  T_ANY, SIP_MEDIA_REMOVABLE|SIP_MEDIA_FIXED,
                  /*vendor*/"*", /*product*/"*", /*revision*/"*"
                },
                /*quirks*/0, /*mintags*/0, /*maxtags*/0
        },
};

static cam_status       aproberegister(struct cam_periph *periph, void *arg);
static void      aprobeschedule(struct cam_periph *probe_periph);
static void      aprobestart(struct cam_periph *periph, union ccb *start_ccb);
static void      aproberequestdefaultnegotiation(struct cam_periph *periph);
static void      aprobedone(struct cam_periph *periph, union ccb *done_ccb);
static void      aprobecleanup(struct cam_periph *periph);
static void      ata_find_quirk(struct cam_ed *device);
static void      ata_scan_bus(struct cam_periph *periph, union ccb *ccb);
static void      ata_scan_lun(struct cam_periph *periph,
                               struct cam_path *path, cam_flags flags,
                               union ccb *ccb);
static void      axptscandone(struct cam_periph *periph, union ccb *done_ccb);
static struct cam_ed *
                 ata_alloc_device(struct cam_eb *bus, struct cam_et *target,
                                   lun_id_t lun_id);
static void      ata_device_transport(struct cam_path *path);
static void      ata_get_transfer_settings(struct ccb_trans_settings *cts);
static void      ata_set_transfer_settings(struct ccb_trans_settings *cts,
                                            struct cam_path *path,
                                            int async_update);
static void      ata_dev_async(uint32_t async_code,
                                struct cam_eb *bus,
                                struct cam_et *target,
                                struct cam_ed *device,
                                void *async_arg);
static void      ata_action(union ccb *start_ccb);
static void      ata_announce_periph_sbuf(struct cam_periph *periph, struct sbuf *sb);
static void      ata_proto_announce_sbuf(struct cam_ed *device, struct sbuf *sb);
static void      ata_proto_denounce_sbuf(struct cam_ed *device, struct sbuf *sb);
static void      ata_proto_debug_out(union ccb *ccb);
static void      semb_proto_announce_sbuf(struct cam_ed *device, struct sbuf *sb);
static void      semb_proto_denounce_sbuf(struct cam_ed *device, struct sbuf *sb);

static int ata_dma = 1;
static int atapi_dma = 1;

TUNABLE_INT("hw.ata.ata_dma", &ata_dma);
TUNABLE_INT("hw.ata.atapi_dma", &atapi_dma);

static struct xpt_xport_ops ata_xport_ops = {
        .alloc_device = ata_alloc_device,
        .action = ata_action,
        .async = ata_dev_async,
        .announce_sbuf = ata_announce_periph_sbuf,
};
#define ATA_XPT_XPORT(x, X)                     \
static struct xpt_xport ata_xport_ ## x = {     \
        .xport = XPORT_ ## X,                   \
        .name = #x,                             \
        .ops = &ata_xport_ops,                  \
};                                              \
CAM_XPT_XPORT(ata_xport_ ## x);

ATA_XPT_XPORT(ata, ATA);
ATA_XPT_XPORT(sata, SATA);

#undef ATA_XPORT_XPORT

static struct xpt_proto_ops ata_proto_ops_ata = {
        .announce_sbuf = ata_proto_announce_sbuf,
        .denounce_sbuf = ata_proto_denounce_sbuf,
        .debug_out = ata_proto_debug_out,
};
static struct xpt_proto ata_proto_ata = {
        .proto = PROTO_ATA,
        .name = "ata",
        .ops = &ata_proto_ops_ata,
};

static struct xpt_proto_ops ata_proto_ops_satapm = {
        .announce_sbuf = ata_proto_announce_sbuf,
        .denounce_sbuf = ata_proto_denounce_sbuf,
        .debug_out = ata_proto_debug_out,
};
static struct xpt_proto ata_proto_satapm = {
        .proto = PROTO_SATAPM,
        .name = "satapm",
        .ops = &ata_proto_ops_satapm,
};

static struct xpt_proto_ops ata_proto_ops_semb = {
        .announce_sbuf = semb_proto_announce_sbuf,
        .denounce_sbuf = semb_proto_denounce_sbuf,
        .debug_out = ata_proto_debug_out,
};
static struct xpt_proto ata_proto_semb = {
        .proto = PROTO_SEMB,
        .name = "semb",
        .ops = &ata_proto_ops_semb,
};

CAM_XPT_PROTO(ata_proto_ata);
CAM_XPT_PROTO(ata_proto_satapm);
CAM_XPT_PROTO(ata_proto_semb);

static void
aprobe_periph_init(void)
{
}

static cam_status
aproberegister(struct cam_periph *periph, void *arg)
{
        union ccb *request_ccb; /* CCB representing the probe request */
        aprobe_softc *softc;

        request_ccb = (union ccb *)arg;
        if (request_ccb == NULL) {
                printf(
                    "proberegister: no probe CCB, can't register device\n");
                return(CAM_REQ_CMP_ERR);
        }

        softc = (aprobe_softc *)malloc(sizeof(*softc), M_CAMXPT, M_ZERO | M_NOWAIT);

        if (softc == NULL) {
                printf(
        "proberegister: Unable to probe new device. Unable to allocate softc\n");
                return(CAM_REQ_CMP_ERR);
        }
        TAILQ_INIT(&softc->request_ccbs);
        TAILQ_INSERT_TAIL(&softc->request_ccbs, &request_ccb->ccb_h,
                          periph_links.tqe);
        softc->flags = 0;
        periph->softc = softc;
        softc->periph = periph;
        softc->action = PROBE_INVALID;
        if (cam_periph_acquire(periph) != 0)
                return (CAM_REQ_CMP_ERR);

        CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe started\n"));
        ata_device_transport(periph->path);
        aprobeschedule(periph);
        return(CAM_REQ_CMP);
}

static void
aprobeschedule(struct cam_periph *periph)
{
        union ccb *ccb;
        aprobe_softc *softc;

        softc = (aprobe_softc *)periph->softc;
        ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs);

        if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) ||
            periph->path->device->protocol == PROTO_SATAPM ||
            periph->path->device->protocol == PROTO_SEMB)
                PROBE_SET_ACTION(softc, PROBE_RESET);
        else
                PROBE_SET_ACTION(softc, PROBE_IDENTIFY);

        if (ccb->crcn.flags & CAM_EXPECT_INQ_CHANGE)
                softc->flags |= PROBE_NO_ANNOUNCE;
        else
                softc->flags &= ~PROBE_NO_ANNOUNCE;

        xpt_schedule(periph, CAM_PRIORITY_XPT);
}

static void
aprobestart(struct cam_periph *periph, union ccb *start_ccb)
{
        struct ccb_trans_settings cts;
        struct ccb_ataio *ataio;
        struct ccb_scsiio *csio;
        aprobe_softc *softc;
        struct cam_path *path;
        struct ata_params *ident_buf;
        u_int oif;

        CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("aprobestart\n"));

        softc = (aprobe_softc *)periph->softc;
        path = start_ccb->ccb_h.path;
        ataio = &start_ccb->ataio;
        csio = &start_ccb->csio;
        ident_buf = &periph->path->device->ident_data;

        if (softc->restart) {
                softc->restart = 0;
                if ((path->device->flags & CAM_DEV_UNCONFIGURED) ||
                    path->device->protocol == PROTO_SATAPM ||
                    path->device->protocol == PROTO_SEMB)
                        softc->action = PROBE_RESET;
                else
                        softc->action = PROBE_IDENTIFY;
        }
        switch (softc->action) {
        case PROBE_RESET:
                cam_fill_ataio(ataio,
                      0,
                      aprobedone,
                      /*flags*/CAM_DIR_NONE,
                      0,
                      /*data_ptr*/NULL,
                      /*dxfer_len*/0,
                      15 * 1000);
                ata_reset_cmd(ataio);
                break;
        case PROBE_IDENTIFY:
                cam_fill_ataio(ataio,
                      1,
                      aprobedone,
                      /*flags*/CAM_DIR_IN,
                      0,
                      /*data_ptr*/(uint8_t *)&softc->ident_data,
                      /*dxfer_len*/sizeof(softc->ident_data),
                      30 * 1000);
                if (path->device->protocol == PROTO_ATA)
                        ata_28bit_cmd(ataio, ATA_ATA_IDENTIFY, 0, 0, 0);
                else
                        ata_28bit_cmd(ataio, ATA_ATAPI_IDENTIFY, 0, 0, 0);
                break;
        case PROBE_SPINUP:
                if (bootverbose)
                        xpt_print(path, "Spinning up device\n");
                cam_fill_ataio(ataio,
                      1,
                      aprobedone,
                      /*flags*/CAM_DIR_NONE | CAM_HIGH_POWER,
                      0,
                      /*data_ptr*/NULL,
                      /*dxfer_len*/0,
                      30 * 1000);
                ata_28bit_cmd(ataio, ATA_SETFEATURES, ATA_SF_PUIS_SPINUP, 0, 0);
                break;
        case PROBE_SETMODE:
        {
                int mode, wantmode;

                mode = 0;
                /* Fetch user modes from SIM. */
                bzero(&cts, sizeof(cts));
                xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
                cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
                cts.type = CTS_TYPE_USER_SETTINGS;
                xpt_action((union ccb *)&cts);
                if (path->device->transport == XPORT_ATA) {
                        if (cts.xport_specific.ata.valid & CTS_ATA_VALID_MODE)
                                mode = cts.xport_specific.ata.mode;
                } else {
                        if (cts.xport_specific.sata.valid & CTS_SATA_VALID_MODE)
                                mode = cts.xport_specific.sata.mode;
                }
                if (path->device->protocol == PROTO_ATA) {
                        if (ata_dma == 0 && (mode == 0 || mode > ATA_PIO_MAX))
                                mode = ATA_PIO_MAX;
                } else {
                        if (atapi_dma == 0 && (mode == 0 || mode > ATA_PIO_MAX))
                                mode = ATA_PIO_MAX;
                }
negotiate:
                /* Honor device capabilities. */
                wantmode = mode = ata_max_mode(ident_buf, mode);
                /* Report modes to SIM. */
                bzero(&cts, sizeof(cts));
                xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
                cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
                cts.type = CTS_TYPE_CURRENT_SETTINGS;
                if (path->device->transport == XPORT_ATA) {
                        cts.xport_specific.ata.mode = mode;
                        cts.xport_specific.ata.valid = CTS_ATA_VALID_MODE;
                } else {
                        cts.xport_specific.sata.mode = mode;
                        cts.xport_specific.sata.valid = CTS_SATA_VALID_MODE;
                }
                xpt_action((union ccb *)&cts);
                /* Fetch current modes from SIM. */
                bzero(&cts, sizeof(cts));
                xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
                cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
                cts.type = CTS_TYPE_CURRENT_SETTINGS;
                xpt_action((union ccb *)&cts);
                if (path->device->transport == XPORT_ATA) {
                        if (cts.xport_specific.ata.valid & CTS_ATA_VALID_MODE)
                                mode = cts.xport_specific.ata.mode;
                } else {
                        if (cts.xport_specific.sata.valid & CTS_SATA_VALID_MODE)
                                mode = cts.xport_specific.sata.mode;
                }
                /* If SIM disagree - renegotiate. */
                if (mode != wantmode)
                        goto negotiate;
                /* Remember what transport thinks about DMA. */
                oif = path->device->inq_flags;
                if (mode < ATA_DMA)
                        path->device->inq_flags &= ~SID_DMA;
                else
                        path->device->inq_flags |= SID_DMA;
                if (path->device->inq_flags != oif)
                        xpt_async(AC_GETDEV_CHANGED, path, NULL);
                cam_fill_ataio(ataio,
                      1,
                      aprobedone,
                      /*flags*/CAM_DIR_NONE,
                      0,
                      /*data_ptr*/NULL,
                      /*dxfer_len*/0,
                      30 * 1000);
                ata_28bit_cmd(ataio, ATA_SETFEATURES, ATA_SF_SETXFER, 0, mode);
                break;
        }
        case PROBE_SETPM:
                cam_fill_ataio(ataio,
                    1,
                    aprobedone,
                    CAM_DIR_NONE,
                    0,
                    NULL,
                    0,
                    30*1000);
                ata_28bit_cmd(ataio, ATA_SETFEATURES,
                    (softc->caps & CTS_SATA_CAPS_H_PMREQ) ? 0x10 : 0x90,
                    0, 0x03);
                break;
        case PROBE_SETAPST:
                cam_fill_ataio(ataio,
                    1,
                    aprobedone,
                    CAM_DIR_NONE,
                    0,
                    NULL,
                    0,
                    30*1000);
                ata_28bit_cmd(ataio, ATA_SETFEATURES,
                    (softc->caps & CTS_SATA_CAPS_H_APST) ? 0x10 : 0x90,
                    0, 0x07);
                break;
        case PROBE_SETDMAAA:
                cam_fill_ataio(ataio,
                    1,
                    aprobedone,
                    CAM_DIR_NONE,
                    0,
                    NULL,
                    0,
                    30*1000);
                ata_28bit_cmd(ataio, ATA_SETFEATURES,
                    (softc->caps & CTS_SATA_CAPS_H_DMAAA) ? 0x10 : 0x90,
                    0, 0x02);
                break;
        case PROBE_SETAN:
                /* Remember what transport thinks about AEN. */
                oif = path->device->inq_flags;
                if (softc->caps & CTS_SATA_CAPS_H_AN)
                        path->device->inq_flags |= SID_AEN;
                else
                        path->device->inq_flags &= ~SID_AEN;
                if (path->device->inq_flags != oif)
                        xpt_async(AC_GETDEV_CHANGED, path, NULL);
                cam_fill_ataio(ataio,
                    1,
                    aprobedone,
                    CAM_DIR_NONE,
                    0,
                    NULL,
                    0,
                    30*1000);
                ata_28bit_cmd(ataio, ATA_SETFEATURES,
                    (softc->caps & CTS_SATA_CAPS_H_AN) ? 0x10 : 0x90,
                    0, 0x05);
                break;
        case PROBE_SET_MULTI:
        {
                u_int sectors, bytecount;

                bytecount = 8192;       /* SATA maximum */
                /* Fetch user bytecount from SIM. */
                bzero(&cts, sizeof(cts));
                xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
                cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
                cts.type = CTS_TYPE_USER_SETTINGS;
                xpt_action((union ccb *)&cts);
                if (path->device->transport == XPORT_ATA) {
                        if (cts.xport_specific.ata.valid & CTS_ATA_VALID_BYTECOUNT)
                                bytecount = cts.xport_specific.ata.bytecount;
                } else {
                        if (cts.xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT)
                                bytecount = cts.xport_specific.sata.bytecount;
                }
                /* Honor device capabilities. */
                sectors = max(1, min(ident_buf->sectors_intr & 0xff,
                    bytecount / ata_logical_sector_size(ident_buf)));
                /* Report bytecount to SIM. */
                bzero(&cts, sizeof(cts));
                xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
                cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
                cts.type = CTS_TYPE_CURRENT_SETTINGS;
                if (path->device->transport == XPORT_ATA) {
                        cts.xport_specific.ata.bytecount = sectors *
                            ata_logical_sector_size(ident_buf);
                        cts.xport_specific.ata.valid = CTS_ATA_VALID_BYTECOUNT;
                } else {
                        cts.xport_specific.sata.bytecount = sectors *
                            ata_logical_sector_size(ident_buf);
                        cts.xport_specific.sata.valid = CTS_SATA_VALID_BYTECOUNT;
                }
                xpt_action((union ccb *)&cts);
                /* Fetch current bytecount from SIM. */
                bzero(&cts, sizeof(cts));
                xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
                cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
                cts.type = CTS_TYPE_CURRENT_SETTINGS;
                xpt_action((union ccb *)&cts);
                if (path->device->transport == XPORT_ATA) {
                        if (cts.xport_specific.ata.valid & CTS_ATA_VALID_BYTECOUNT)
                                bytecount = cts.xport_specific.ata.bytecount;
                } else {
                        if (cts.xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT)
                                bytecount = cts.xport_specific.sata.bytecount;
                }
                sectors = bytecount / ata_logical_sector_size(ident_buf);

                cam_fill_ataio(ataio,
                    1,
                    aprobedone,
                    CAM_DIR_NONE,
                    0,
                    NULL,
                    0,
                    30*1000);
                ata_28bit_cmd(ataio, ATA_SET_MULTI, 0, 0, sectors);
                break;
        }
        case PROBE_INQUIRY:
        {
                u_int bytecount;

                bytecount = 8192;       /* SATA maximum */
                /* Fetch user bytecount from SIM. */
                bzero(&cts, sizeof(cts));
                xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
                cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
                cts.type = CTS_TYPE_USER_SETTINGS;
                xpt_action((union ccb *)&cts);
                if (path->device->transport == XPORT_ATA) {
                        if (cts.xport_specific.ata.valid & CTS_ATA_VALID_BYTECOUNT)
                                bytecount = cts.xport_specific.ata.bytecount;
                } else {
                        if (cts.xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT)
                                bytecount = cts.xport_specific.sata.bytecount;
                }
                /* Honor device capabilities. */
                bytecount &= ~1;
                bytecount = max(2, min(65534, bytecount));
                if (ident_buf->satacapabilities != 0x0000 &&
                    ident_buf->satacapabilities != 0xffff) {
                        bytecount = min(8192, bytecount);
                }
                /* Report bytecount to SIM. */
                bzero(&cts, sizeof(cts));
                xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
                cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
                cts.type = CTS_TYPE_CURRENT_SETTINGS;
                if (path->device->transport == XPORT_ATA) {
                        cts.xport_specific.ata.bytecount = bytecount;
                        cts.xport_specific.ata.valid = CTS_ATA_VALID_BYTECOUNT;
                } else {
                        cts.xport_specific.sata.bytecount = bytecount;
                        cts.xport_specific.sata.valid = CTS_SATA_VALID_BYTECOUNT;
                }
                xpt_action((union ccb *)&cts);
                /* FALLTHROUGH */
        }
        case PROBE_FULL_INQUIRY:
        {
                u_int inquiry_len;
                struct scsi_inquiry_data *inq_buf =
                    &path->device->inq_data;

                if (softc->action == PROBE_INQUIRY)
                        inquiry_len = SHORT_INQUIRY_LENGTH;
                else
                        inquiry_len = SID_ADDITIONAL_LENGTH(inq_buf);
                /*
                 * Some parallel SCSI devices fail to send an
                 * ignore wide residue message when dealing with
                 * odd length inquiry requests.  Round up to be
                 * safe.
                 */
                inquiry_len = roundup2(inquiry_len, 2);
                scsi_inquiry(csio,
                             /*retries*/1,
                             aprobedone,
                             MSG_SIMPLE_Q_TAG,
                             (uint8_t *)inq_buf,
                             inquiry_len,
                             /*evpd*/FALSE,
                             /*page_code*/0,
                             SSD_MIN_SIZE,
                             /*timeout*/60 * 1000);
                break;
        }
        case PROBE_PM_PID:
                cam_fill_ataio(ataio,
                      1,
                      aprobedone,
                      /*flags*/CAM_DIR_NONE,
                      0,
                      /*data_ptr*/NULL,
                      /*dxfer_len*/0,
                      10 * 1000);
                ata_pm_read_cmd(ataio, 0, 15);
                break;
        case PROBE_PM_PRV:
                cam_fill_ataio(ataio,
                      1,
                      aprobedone,
                      /*flags*/CAM_DIR_NONE,
                      0,
                      /*data_ptr*/NULL,
                      /*dxfer_len*/0,
                      10 * 1000);
                ata_pm_read_cmd(ataio, 1, 15);
                break;
        case PROBE_IDENTIFY_SES:
                cam_fill_ataio(ataio,
                      1,
                      aprobedone,
                      /*flags*/CAM_DIR_IN,
                      0,
                      /*data_ptr*/(uint8_t *)&softc->ident_data,
                      /*dxfer_len*/sizeof(softc->ident_data),
                      30 * 1000);
                ata_28bit_cmd(ataio, ATA_SEP_ATTN, 0xEC, 0x02,
                    sizeof(softc->ident_data) / 4);
                break;
        case PROBE_IDENTIFY_SAFTE:
                cam_fill_ataio(ataio,
                      1,
                      aprobedone,
                      /*flags*/CAM_DIR_IN,
                      0,
                      /*data_ptr*/(uint8_t *)&softc->ident_data,
                      /*dxfer_len*/sizeof(softc->ident_data),
                      30 * 1000);
                ata_28bit_cmd(ataio, ATA_SEP_ATTN, 0xEC, 0x00,
                    sizeof(softc->ident_data) / 4);
                break;
        default:
                panic("aprobestart: invalid action state 0x%x\n", softc->action);
        }
        start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
        xpt_action(start_ccb);
}

static void
aproberequestdefaultnegotiation(struct cam_periph *periph)
{
        struct ccb_trans_settings cts;

        bzero(&cts, sizeof(cts));
        xpt_setup_ccb(&cts.ccb_h, periph->path, CAM_PRIORITY_NONE);
        cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
        cts.type = CTS_TYPE_USER_SETTINGS;
        xpt_action((union ccb *)&cts);
        if ((cts.ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
                return;
        cts.xport_specific.valid = 0;
        cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
        cts.type = CTS_TYPE_CURRENT_SETTINGS;
        xpt_action((union ccb *)&cts);
}

static void
aprobedone(struct cam_periph *periph, union ccb *done_ccb)
{
        struct ccb_trans_settings cts;
        struct ata_params *ident_buf;
        struct scsi_inquiry_data *inq_buf;
        aprobe_softc *softc;
        struct cam_path *path;
        cam_status status;
        uint32_t  priority;
        u_int caps, oif;
        int changed, found = 1;
        static const uint8_t fake_device_id_hdr[8] =
            {0, SVPD_DEVICE_ID, 0, 12,
             SVPD_ID_CODESET_BINARY, SVPD_ID_TYPE_NAA, 0, 8};

        CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("aprobedone\n"));

        softc = (aprobe_softc *)periph->softc;
        path = done_ccb->ccb_h.path;
        priority = done_ccb->ccb_h.pinfo.priority;
        ident_buf = &path->device->ident_data;
        inq_buf = &path->device->inq_data;

        if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
                if (cam_periph_error(done_ccb,
                        0, softc->restart ? (SF_NO_RECOVERY | SF_NO_RETRY) : 0
                    ) == ERESTART) {
out:
                        /* Drop freeze taken due to CAM_DEV_QFREEZE flag set. */
                        cam_release_devq(path, 0, 0, 0, FALSE);
                        return;
                }
                if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
                        /* Don't wedge the queue */
                        xpt_release_devq(path, /*count*/1, /*run_queue*/TRUE);
                }
                status = done_ccb->ccb_h.status & CAM_STATUS_MASK;
                if (softc->restart) {
                        softc->faults++;
                        if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) ==
                            CAM_CMD_TIMEOUT)
                                softc->faults += 4;
                        if (softc->faults < 10)
                                goto done;
                        else
                                softc->restart = 0;

                /* Old PIO2 devices may not support mode setting. */
                } else if (softc->action == PROBE_SETMODE &&
                    status == CAM_ATA_STATUS_ERROR &&
                    ata_max_pmode(ident_buf) <= ATA_PIO2 &&
                    (ident_buf->capabilities1 & ATA_SUPPORT_IORDY) == 0) {
                        goto noerror;

                /*
                 * Some old WD SATA disks report supported and enabled
                 * device-initiated interface power management, but return
                 * ABORT on attempt to disable it.
                 */
                } else if (softc->action == PROBE_SETPM &&
                    status == CAM_ATA_STATUS_ERROR) {
                        goto noerror;

                /*
                 * Some old WD SATA disks have broken SPINUP handling.
                 * If we really fail to spin up the disk, then there will be
                 * some media access errors later on, but at least we will
                 * have a device to interact with for recovery attempts.
                 */
                } else if (softc->action == PROBE_SPINUP &&
                    status == CAM_ATA_STATUS_ERROR) {
                        goto noerror;

                /*
                 * Some HP SATA disks report supported DMA Auto-Activation,
                 * but return ABORT on attempt to enable it.
                 */
                } else if (softc->action == PROBE_SETDMAAA &&
                    status == CAM_ATA_STATUS_ERROR) {
                        goto noerror;

                /*
                 * SES and SAF-TE SEPs have different IDENTIFY commands,
                 * but SATA specification doesn't tell how to identify them.
                 * Until better way found, just try another if first fail.
                 */
                } else if (softc->action == PROBE_IDENTIFY_SES &&
                    status == CAM_ATA_STATUS_ERROR) {
                        PROBE_SET_ACTION(softc, PROBE_IDENTIFY_SAFTE);
                        xpt_release_ccb(done_ccb);
                        xpt_schedule(periph, priority);
                        goto out;
                }

                /*
                 * If we get to this point, we got an error status back
                 * from the inquiry and the error status doesn't require
                 * automatically retrying the command.  Therefore, the
                 * inquiry failed.  If we had inquiry information before
                 * for this device, but this latest inquiry command failed,
                 * the device has probably gone away.  If this device isn't
                 * already marked unconfigured, notify the peripheral
                 * drivers that this device is no more.
                 */
device_fail:    if ((path->device->flags & CAM_DEV_UNCONFIGURED) == 0)
                        xpt_async(AC_LOST_DEVICE, path, NULL);
                PROBE_SET_ACTION(softc, PROBE_INVALID);
                found = 0;
                goto done;
        }
noerror:
        if (softc->restart)
                goto done;
        switch (softc->action) {
        case PROBE_RESET:
        {
                int sign = (done_ccb->ataio.res.lba_high << 8) +
                    done_ccb->ataio.res.lba_mid;
                CAM_DEBUG(path, CAM_DEBUG_PROBE,
                    ("SIGNATURE: %04x\n", sign));
                if (sign == 0x0000 &&
                    done_ccb->ccb_h.target_id != 15) {
                        path->device->protocol = PROTO_ATA;
                        PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
                } else if (sign == 0x9669 &&
                    done_ccb->ccb_h.target_id == 15) {
                        /* Report SIM that PM is present. */
                        bzero(&cts, sizeof(cts));
                        xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
                        cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
                        cts.type = CTS_TYPE_CURRENT_SETTINGS;
                        cts.xport_specific.sata.pm_present = 1;
                        cts.xport_specific.sata.valid = CTS_SATA_VALID_PM;
                        xpt_action((union ccb *)&cts);
                        path->device->protocol = PROTO_SATAPM;
                        PROBE_SET_ACTION(softc, PROBE_PM_PID);
                } else if (sign == 0xc33c &&
                    done_ccb->ccb_h.target_id != 15) {
                        path->device->protocol = PROTO_SEMB;
                        PROBE_SET_ACTION(softc, PROBE_IDENTIFY_SES);
                } else if (sign == 0xeb14 &&
                    done_ccb->ccb_h.target_id != 15) {
                        path->device->protocol = PROTO_SCSI;
                        PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
                } else {
                        if (done_ccb->ccb_h.target_id != 15) {
                                xpt_print(path,
                                    "Unexpected signature 0x%04x\n", sign);
                        }
                        goto device_fail;
                }
                xpt_release_ccb(done_ccb);
                xpt_schedule(periph, priority);
                goto out;
        }
        case PROBE_IDENTIFY:
        {
                struct ccb_pathinq cpi;
                int veto = 0;

                /*
                 * Convert to host byte order, and fix the strings.
                 */
                ident_buf = &softc->ident_data;
                ata_param_fixup(ident_buf);

                /*
                 * Allow others to veto this ATA disk attachment.  This
                 * is mainly used by VMs, whose disk controllers may
                 * share the disks with the simulated ATA controllers.
                 */
                EVENTHANDLER_INVOKE(ada_probe_veto, path, ident_buf, &veto);
                if (veto) {
                        goto device_fail;
                }

                /* Device may need spin-up before IDENTIFY become valid. */
                if ((ident_buf->specconf == 0x37c8 ||
                     ident_buf->specconf == 0x738c) &&
                    ((ident_buf->config & ATA_RESP_INCOMPLETE) ||
                     softc->spinup == 0)) {
                        PROBE_SET_ACTION(softc, PROBE_SPINUP);
                        xpt_release_ccb(done_ccb);
                        xpt_schedule(periph, priority);
                        goto out;
                }
                ident_buf = &path->device->ident_data;

                /* Check that it is the same device as we know. */
                if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) == 0) {
                        if (bcmp(softc->ident_data.model, ident_buf->model,
                             sizeof(ident_buf->model)) ||
                            bcmp(softc->ident_data.serial, ident_buf->serial,
                             sizeof(ident_buf->serial))) {
                                /* The device was replaced. */
                                changed = 2;
                                xpt_async(AC_LOST_DEVICE, path, NULL);
                        } else if (bcmp(&softc->ident_data, ident_buf,
                             sizeof(*ident_buf))) {
                                /* The device is the same, but has changed. */
                                changed = 1;
                        } else {
                                /* Nothing has changed. */
                                changed = 0;
                        }
                } else {
                        /* This is a new device. */
                        changed = 2;
                }

                if (changed != 0)
                        bcopy(&softc->ident_data, ident_buf, sizeof(struct ata_params));
                if (changed == 2) {
                        /* Clean up from previous instance of this device */
                        if (path->device->serial_num != NULL) {
                                free(path->device->serial_num, M_CAMXPT);
                                path->device->serial_num = NULL;
                                path->device->serial_num_len = 0;
                        }
                        if (path->device->device_id != NULL) {
                                free(path->device->device_id, M_CAMXPT);
                                path->device->device_id = NULL;
                                path->device->device_id_len = 0;
                        }
                        path->device->serial_num =
                                (uint8_t *)malloc((sizeof(ident_buf->serial) + 1),
                                           M_CAMXPT, M_NOWAIT);
                        if (path->device->serial_num != NULL) {
                                bcopy(ident_buf->serial,
                                      path->device->serial_num,
                                      sizeof(ident_buf->serial));
                                path->device->serial_num[sizeof(ident_buf->serial)]
                                    = '\0';
                                path->device->serial_num_len =
                                    strlen(path->device->serial_num);
                        }
                        if (ident_buf->enabled.extension &
                            ATA_SUPPORT_64BITWWN) {
                                path->device->device_id =
                                    malloc(16, M_CAMXPT, M_NOWAIT);
                                if (path->device->device_id != NULL) {
                                        path->device->device_id_len = 16;
                                        bcopy(&fake_device_id_hdr,
                                            path->device->device_id, 8);
                                        bcopy(ident_buf->wwn,
                                            path->device->device_id + 8, 8);
                                        ata_bswap(path->device->device_id + 8, 8);
                                }
                        }
                        path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID;
                }
                if (changed == 1)
                        xpt_async(AC_GETDEV_CHANGED, path, NULL);
                if (ident_buf->satacapabilities & ATA_SUPPORT_NCQ) {
                        path->device->mintags = 2;
                        path->device->maxtags =
                            ATA_QUEUE_LEN(ident_buf->queue) + 1;
                }
                ata_find_quirk(path->device);
                if (path->device->mintags != 0 &&
                    path->bus->sim->max_tagged_dev_openings != 0) {
                        /* Check if the SIM does not want queued commands. */
                        xpt_path_inq(&cpi, path);
                        if (cam_ccb_success((union ccb *)&cpi) &&
                            (cpi.hba_inquiry & PI_TAG_ABLE)) {
                                /* Report SIM which tags are allowed. */
                                bzero(&cts, sizeof(cts));
                                xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
                                cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
                                cts.type = CTS_TYPE_CURRENT_SETTINGS;
                                cts.xport_specific.sata.tags = path->device->maxtags;
                                cts.xport_specific.sata.valid = CTS_SATA_VALID_TAGS;
                                xpt_action((union ccb *)&cts);
                        }
                }
                ata_device_transport(path);
                if (changed == 2)
                        aproberequestdefaultnegotiation(periph);
                PROBE_SET_ACTION(softc, PROBE_SETMODE);
                xpt_release_ccb(done_ccb);
                xpt_schedule(periph, priority);
                goto out;
        }
        case PROBE_SPINUP:
                if (bootverbose)
                        xpt_print(path, "Spin-up done\n");
                softc->spinup = 1;
                PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
                xpt_release_ccb(done_ccb);
                xpt_schedule(periph, priority);
                goto out;
        case PROBE_SETMODE:
                /* Set supported bits. */
                bzero(&cts, sizeof(cts));
                xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
                cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
                cts.type = CTS_TYPE_CURRENT_SETTINGS;
                xpt_action((union ccb *)&cts);
                if (path->device->transport == XPORT_SATA &&
                    cts.xport_specific.sata.valid & CTS_SATA_VALID_CAPS)
                        caps = cts.xport_specific.sata.caps & CTS_SATA_CAPS_H;
                else if (path->device->transport == XPORT_ATA &&
                    cts.xport_specific.ata.valid & CTS_ATA_VALID_CAPS)
                        caps = cts.xport_specific.ata.caps & CTS_ATA_CAPS_H;
                else
                        caps = 0;
                if (path->device->transport == XPORT_SATA &&
                    ident_buf->satacapabilities != 0xffff) {
                        if (ident_buf->satacapabilities & ATA_SUPPORT_IFPWRMNGTRCV)
                                caps |= CTS_SATA_CAPS_D_PMREQ;
                        if (ident_buf->satacapabilities & ATA_SUPPORT_HAPST)
                                caps |= CTS_SATA_CAPS_D_APST;
                }
                /* Mask unwanted bits. */
                bzero(&cts, sizeof(cts));
                xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
                cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
                cts.type = CTS_TYPE_USER_SETTINGS;
                xpt_action((union ccb *)&cts);
                if (path->device->transport == XPORT_SATA &&
                    cts.xport_specific.sata.valid & CTS_SATA_VALID_CAPS)
                        caps &= cts.xport_specific.sata.caps;
                else if (path->device->transport == XPORT_ATA &&
                    cts.xport_specific.ata.valid & CTS_ATA_VALID_CAPS)
                        caps &= cts.xport_specific.ata.caps;
                else
                        caps = 0;
                /*
                 * Remember what transport thinks about 48-bit DMA.  If
                 * capability information is not provided or transport is
                 * SATA, we take support for granted.
                 */
                oif = path->device->inq_flags;
                if (!(path->device->inq_flags & SID_DMA) ||
                    (path->device->transport == XPORT_ATA &&
                    (cts.xport_specific.ata.valid & CTS_ATA_VALID_CAPS) &&
                    !(caps & CTS_ATA_CAPS_H_DMA48)))
                        path->device->inq_flags &= ~SID_DMA48;
                else
                        path->device->inq_flags |= SID_DMA48;
                if (path->device->inq_flags != oif)
                        xpt_async(AC_GETDEV_CHANGED, path, NULL);
                /* Store result to SIM. */
                bzero(&cts, sizeof(cts));
                xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
                cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
                cts.type = CTS_TYPE_CURRENT_SETTINGS;
                if (path->device->transport == XPORT_SATA) {
                        cts.xport_specific.sata.caps = caps;
                        cts.xport_specific.sata.valid = CTS_SATA_VALID_CAPS;
                } else {
                        cts.xport_specific.ata.caps = caps;
                        cts.xport_specific.ata.valid = CTS_ATA_VALID_CAPS;
                }
                xpt_action((union ccb *)&cts);
                softc->caps = caps;
                if (path->device->transport != XPORT_SATA)
                        goto notsata;
                if ((ident_buf->satasupport & ATA_SUPPORT_IFPWRMNGT) &&
                    (!(softc->caps & CTS_SATA_CAPS_H_PMREQ)) !=
                    (!(ident_buf->sataenabled & ATA_SUPPORT_IFPWRMNGT))) {
                        PROBE_SET_ACTION(softc, PROBE_SETPM);
                        xpt_release_ccb(done_ccb);
                        xpt_schedule(periph, priority);
                        goto out;
                }
                /* FALLTHROUGH */
        case PROBE_SETPM:
                if (ident_buf->satacapabilities != 0xffff &&
                    (ident_buf->satacapabilities & ATA_SUPPORT_DAPST) &&
                    (!(softc->caps & CTS_SATA_CAPS_H_APST)) !=
                    (!(ident_buf->sataenabled & ATA_ENABLED_DAPST))) {
                        PROBE_SET_ACTION(softc, PROBE_SETAPST);
                        xpt_release_ccb(done_ccb);
                        xpt_schedule(periph, priority);
                        goto out;
                }
                /* FALLTHROUGH */
        case PROBE_SETAPST:
                if ((ident_buf->satasupport & ATA_SUPPORT_AUTOACTIVATE) &&
                    (!(softc->caps & CTS_SATA_CAPS_H_DMAAA)) !=
                    (!(ident_buf->sataenabled & ATA_SUPPORT_AUTOACTIVATE))) {
                        PROBE_SET_ACTION(softc, PROBE_SETDMAAA);
                        xpt_release_ccb(done_ccb);
                        xpt_schedule(periph, priority);
                        goto out;
                }
                /* FALLTHROUGH */
        case PROBE_SETDMAAA:
                if (path->device->protocol != PROTO_ATA &&
                    (ident_buf->satasupport & ATA_SUPPORT_ASYNCNOTIF) &&
                    (!(softc->caps & CTS_SATA_CAPS_H_AN)) !=
                    (!(ident_buf->sataenabled & ATA_SUPPORT_ASYNCNOTIF))) {
                        PROBE_SET_ACTION(softc, PROBE_SETAN);
                        xpt_release_ccb(done_ccb);
                        xpt_schedule(periph, priority);
                        goto out;
                }
                /* FALLTHROUGH */
        case PROBE_SETAN:
notsata:
                if (path->device->protocol == PROTO_ATA) {
                        PROBE_SET_ACTION(softc, PROBE_SET_MULTI);
                } else {
                        PROBE_SET_ACTION(softc, PROBE_INQUIRY);
                }
                xpt_release_ccb(done_ccb);
                xpt_schedule(periph, priority);
                goto out;
        case PROBE_SET_MULTI:
                if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
                        path->device->flags &= ~CAM_DEV_UNCONFIGURED;
                        xpt_acquire_device(path->device);
                        done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
                        xpt_action(done_ccb);
                        xpt_async(AC_FOUND_DEVICE, path, done_ccb);
                }
                PROBE_SET_ACTION(softc, PROBE_DONE);
                break;
        case PROBE_INQUIRY:
        case PROBE_FULL_INQUIRY:
        {
                uint8_t periph_qual, len;

                path->device->flags |= CAM_DEV_INQUIRY_DATA_VALID;

                periph_qual = SID_QUAL(inq_buf);

                if (periph_qual != SID_QUAL_LU_CONNECTED &&
                    periph_qual != SID_QUAL_LU_OFFLINE)
                        break;

                /*
                 * We conservatively request only
                 * SHORT_INQUIRY_LEN bytes of inquiry
                 * information during our first try
                 * at sending an INQUIRY. If the device
                 * has more information to give,
                 * perform a second request specifying
                 * the amount of information the device
                 * is willing to give.
                 */
                len = inq_buf->additional_length
                    + offsetof(struct scsi_inquiry_data, additional_length) + 1;
                if (softc->action == PROBE_INQUIRY
                    && len > SHORT_INQUIRY_LENGTH) {
                        PROBE_SET_ACTION(softc, PROBE_FULL_INQUIRY);
                        xpt_release_ccb(done_ccb);
                        xpt_schedule(periph, priority);
                        goto out;
                }

                ata_device_transport(path);
                if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
                        path->device->flags &= ~CAM_DEV_UNCONFIGURED;
                        xpt_acquire_device(path->device);
                        done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
                        xpt_action(done_ccb);
                        xpt_async(AC_FOUND_DEVICE, path, done_ccb);
                }
                PROBE_SET_ACTION(softc, PROBE_DONE);
                break;
        }
        case PROBE_PM_PID:
                if ((path->device->flags & CAM_DEV_IDENTIFY_DATA_VALID) == 0)
                        bzero(ident_buf, sizeof(*ident_buf));
                softc->pm_pid = (done_ccb->ataio.res.lba_high << 24) +
                    (done_ccb->ataio.res.lba_mid << 16) +
                    (done_ccb->ataio.res.lba_low << 8) +
                    done_ccb->ataio.res.sector_count;
                ((uint32_t *)ident_buf)[0] = softc->pm_pid;
                snprintf(ident_buf->model, sizeof(ident_buf->model),
                    "Port Multiplier %08x", softc->pm_pid);
                PROBE_SET_ACTION(softc, PROBE_PM_PRV);
                xpt_release_ccb(done_ccb);
                xpt_schedule(periph, priority);
                goto out;
        case PROBE_PM_PRV:
                softc->pm_prv = (done_ccb->ataio.res.lba_high << 24) +
                    (done_ccb->ataio.res.lba_mid << 16) +
                    (done_ccb->ataio.res.lba_low << 8) +
                    done_ccb->ataio.res.sector_count;
                ((uint32_t *)ident_buf)[1] = softc->pm_prv;
                snprintf(ident_buf->revision, sizeof(ident_buf->revision),
                    "%04x", softc->pm_prv);
                path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID;
                ata_device_transport(path);
                if (periph->path->device->flags & CAM_DEV_UNCONFIGURED)
                        aproberequestdefaultnegotiation(periph);
                /* Set supported bits. */
                bzero(&cts, sizeof(cts));
                xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
                cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
                cts.type = CTS_TYPE_CURRENT_SETTINGS;
                xpt_action((union ccb *)&cts);
                if (cts.xport_specific.sata.valid & CTS_SATA_VALID_CAPS)
                        caps = cts.xport_specific.sata.caps & CTS_SATA_CAPS_H;
                else
                        caps = 0;
                /* All PMPs must support PM requests. */
                caps |= CTS_SATA_CAPS_D_PMREQ;
                /* Mask unwanted bits. */
                bzero(&cts, sizeof(cts));
                xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
                cts.ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
                cts.type = CTS_TYPE_USER_SETTINGS;
                xpt_action((union ccb *)&cts);
                if (cts.xport_specific.sata.valid & CTS_SATA_VALID_CAPS)
                        caps &= cts.xport_specific.sata.caps;
                else
                        caps = 0;
                /* Remember what transport thinks about AEN. */
                oif = path->device->inq_flags;
                if ((caps & CTS_SATA_CAPS_H_AN) && path->device->protocol != PROTO_ATA)
                        path->device->inq_flags |= SID_AEN;
                else
                        path->device->inq_flags &= ~SID_AEN;
                /* Store result to SIM. */
                bzero(&cts, sizeof(cts));
                xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
                cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
                cts.type = CTS_TYPE_CURRENT_SETTINGS;
                cts.xport_specific.sata.caps = caps;
                cts.xport_specific.sata.valid = CTS_SATA_VALID_CAPS;
                xpt_action((union ccb *)&cts);
                softc->caps = caps;
                if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
                        path->device->flags &= ~CAM_DEV_UNCONFIGURED;
                        xpt_acquire_device(path->device);
                        done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
                        xpt_action(done_ccb);
                        xpt_async(AC_FOUND_DEVICE, path, done_ccb);
                } else {
                        if (path->device->inq_flags != oif)
                                xpt_async(AC_GETDEV_CHANGED, path, NULL);
                        done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
                        xpt_action(done_ccb);
                        xpt_async(AC_SCSI_AEN, path, done_ccb);
                }
                PROBE_SET_ACTION(softc, PROBE_DONE);
                break;
        case PROBE_IDENTIFY_SES:
        case PROBE_IDENTIFY_SAFTE:
                if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) == 0) {
                        /* Check that it is the same device. */
                        if (bcmp(&softc->ident_data, ident_buf, 53)) {
                                /* Device changed. */
                                changed = 2;
                                xpt_async(AC_LOST_DEVICE, path, NULL);
                        } else {
                                bcopy(&softc->ident_data, ident_buf, sizeof(struct ata_params));
                                changed = 0;
                        }
                } else
                        changed = 2;
                if (changed) {
                        bcopy(&softc->ident_data, ident_buf, sizeof(struct ata_params));
                        /* Clean up from previous instance of this device */
                        if (path->device->device_id != NULL) {
                                free(path->device->device_id, M_CAMXPT);
                                path->device->device_id = NULL;
                                path->device->device_id_len = 0;
                        }
                        path->device->device_id =
                            malloc(16, M_CAMXPT, M_NOWAIT);
                        if (path->device->device_id != NULL) {
                                path->device->device_id_len = 16;
                                bcopy(&fake_device_id_hdr,
                                    path->device->device_id, 8);
                                bcopy(((uint8_t*)ident_buf) + 2,
                                    path->device->device_id + 8, 8);
                        }

                        path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID;
                }
                ata_device_transport(path);
                if (changed)
                        aproberequestdefaultnegotiation(periph);

                if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
                        path->device->flags &= ~CAM_DEV_UNCONFIGURED;
                        xpt_acquire_device(path->device);
                        done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
                        xpt_action(done_ccb);
                        xpt_async(AC_FOUND_DEVICE, path, done_ccb);
                }
                PROBE_SET_ACTION(softc, PROBE_DONE);
                break;
        default:
                panic("aprobedone: invalid action state 0x%x\n", softc->action);
        }
done:
        if (softc->restart) {
                softc->restart = 0;
                xpt_release_ccb(done_ccb);
                aprobeschedule(periph);
                goto out;
        }
        xpt_release_ccb(done_ccb);
        CAM_DEBUG(periph->path, CAM_DEBUG_PROBE, ("Probe completed\n"));
        while ((done_ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs))) {
                TAILQ_REMOVE(&softc->request_ccbs,
                    &done_ccb->ccb_h, periph_links.tqe);
                done_ccb->ccb_h.status = found ? CAM_REQ_CMP : CAM_REQ_CMP_ERR;
                xpt_done(done_ccb);
        }
        /* Drop freeze taken due to CAM_DEV_QFREEZE flag set. */
        cam_release_devq(path, 0, 0, 0, FALSE);
        cam_periph_invalidate(periph);
        cam_periph_release_locked(periph);
}

static void
aprobecleanup(struct cam_periph *periph)
{
        free(periph->softc, M_CAMXPT);
}

static void
ata_find_quirk(struct cam_ed *device)
{
        struct ata_quirk_entry *quirk;
        caddr_t match;

        match = cam_quirkmatch((caddr_t)&device->ident_data,
                               (caddr_t)ata_quirk_table,
                               nitems(ata_quirk_table),
                               sizeof(*ata_quirk_table), ata_identify_match);

        if (match == NULL)
                panic("xpt_find_quirk: device didn't match wildcard entry!!");

        quirk = (struct ata_quirk_entry *)match;
        device->quirk = quirk;
        if (quirk->quirks & CAM_QUIRK_MAXTAGS) {
                device->mintags = quirk->mintags;
                device->maxtags = quirk->maxtags;
        }
}

typedef struct {
        union   ccb *request_ccb;
        struct  ccb_pathinq *cpi;
        int     counter;
} ata_scan_bus_info;

/*
 * To start a scan, request_ccb is an XPT_SCAN_BUS ccb.
 * As the scan progresses, xpt_scan_bus is used as the
 * callback on completion function.
 */
static void
ata_scan_bus(struct cam_periph *periph, union ccb *request_ccb)
{
        struct  cam_path *path;
        ata_scan_bus_info *scan_info;
        union   ccb *work_ccb, *reset_ccb;
        struct mtx *mtx;
        cam_status status;

        CAM_DEBUG(request_ccb->ccb_h.path, CAM_DEBUG_TRACE,
                  ("xpt_scan_bus\n"));
        switch (request_ccb->ccb_h.func_code) {
        case XPT_SCAN_BUS:
        case XPT_SCAN_TGT:
                /* Find out the characteristics of the bus */
                work_ccb = xpt_alloc_ccb_nowait();
                if (work_ccb == NULL) {
                        request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
                        xpt_done(request_ccb);
                        return;
                }
                xpt_path_inq(&work_ccb->cpi, request_ccb->ccb_h.path);
                if (work_ccb->ccb_h.status != CAM_REQ_CMP) {
                        request_ccb->ccb_h.status = work_ccb->ccb_h.status;
                        xpt_free_ccb(work_ccb);
                        xpt_done(request_ccb);
                        return;
                }

                /* We may need to reset bus first, if we haven't done it yet. */
                if ((work_ccb->cpi.hba_inquiry &
                    (PI_WIDE_32|PI_WIDE_16|PI_SDTR_ABLE)) &&
                    !(work_ccb->cpi.hba_misc & PIM_NOBUSRESET) &&
                    !timevalisset(&request_ccb->ccb_h.path->bus->last_reset)) {
                        reset_ccb = xpt_alloc_ccb_nowait();
                        if (reset_ccb == NULL) {
                                request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
                                xpt_free_ccb(work_ccb);
                                xpt_done(request_ccb);
                                return;
                        }
                        xpt_setup_ccb(&reset_ccb->ccb_h, request_ccb->ccb_h.path,
                              CAM_PRIORITY_NONE);
                        reset_ccb->ccb_h.func_code = XPT_RESET_BUS;
                        xpt_action(reset_ccb);
                        if (reset_ccb->ccb_h.status != CAM_REQ_CMP) {
                                request_ccb->ccb_h.status = reset_ccb->ccb_h.status;
                                xpt_free_ccb(reset_ccb);
                                xpt_free_ccb(work_ccb);
                                xpt_done(request_ccb);
                                return;
                        }
                        xpt_free_ccb(reset_ccb);
                }

                /* Save some state for use while we probe for devices */
                scan_info = (ata_scan_bus_info *)
                    malloc(sizeof(ata_scan_bus_info), M_CAMXPT, M_NOWAIT);
                if (scan_info == NULL) {
                        request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
                        xpt_free_ccb(work_ccb);
                        xpt_done(request_ccb);
                        return;
                }
                scan_info->request_ccb = request_ccb;
                scan_info->cpi = &work_ccb->cpi;
                /* If PM supported, probe it first. */
                if (scan_info->cpi->hba_inquiry & PI_SATAPM)
                        scan_info->counter = scan_info->cpi->max_target;
                else
                        scan_info->counter = 0;

                work_ccb = xpt_alloc_ccb_nowait();
                if (work_ccb == NULL) {
                        free(scan_info, M_CAMXPT);
                        request_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
                        xpt_done(request_ccb);
                        break;
                }
                mtx = xpt_path_mtx(scan_info->request_ccb->ccb_h.path);
                goto scan_next;
        case XPT_SCAN_LUN:
                work_ccb = request_ccb;
                /* Reuse the same CCB to query if a device was really found */
                scan_info = (ata_scan_bus_info *)work_ccb->ccb_h.ppriv_ptr0;
                mtx = xpt_path_mtx(scan_info->request_ccb->ccb_h.path);
                mtx_lock(mtx);
                /* If there is PMP... */
                if ((scan_info->cpi->hba_inquiry & PI_SATAPM) &&
                    (scan_info->counter == scan_info->cpi->max_target)) {
                        if (cam_ccb_success(work_ccb)) {
                                /* everything else will be probed by it */
                                /* Free the current request path- we're done with it. */
                                xpt_free_path(work_ccb->ccb_h.path);
                                goto done;
                        } else {
                                struct ccb_trans_settings cts;

                                /* Report SIM that PM is absent. */
                                bzero(&cts, sizeof(cts));
                                xpt_setup_ccb(&cts.ccb_h,
                                    work_ccb->ccb_h.path, CAM_PRIORITY_NONE);
                                cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
                                cts.type = CTS_TYPE_CURRENT_SETTINGS;
                                cts.xport_specific.sata.pm_present = 0;
                                cts.xport_specific.sata.valid = CTS_SATA_VALID_PM;
                                xpt_action((union ccb *)&cts);
                        }
                }
                /* Free the current request path- we're done with it. */
                xpt_free_path(work_ccb->ccb_h.path);
                if (scan_info->counter ==
                    ((scan_info->cpi->hba_inquiry & PI_SATAPM) ?
                    0 : scan_info->cpi->max_target)) {
done:
                        mtx_unlock(mtx);
                        xpt_free_ccb(work_ccb);
                        xpt_free_ccb((union ccb *)scan_info->cpi);
                        request_ccb = scan_info->request_ccb;
                        free(scan_info, M_CAMXPT);
                        request_ccb->ccb_h.status = CAM_REQ_CMP;
                        xpt_done(request_ccb);
                        break;
                }
                /* Take next device. Wrap from max (PMP) to 0. */
                scan_info->counter = (scan_info->counter + 1 ) %
                    (scan_info->cpi->max_target + 1);
scan_next:
                status = xpt_create_path(&path, NULL,
                    scan_info->request_ccb->ccb_h.path_id,
                    scan_info->counter, 0);
                if (status != CAM_REQ_CMP) {
                        if (request_ccb->ccb_h.func_code == XPT_SCAN_LUN)
                                mtx_unlock(mtx);
                        printf(
        "xpt_scan_bus: xpt_create_path failed with status %#x, bus scan halted\n",
                            status);
                        xpt_free_ccb(work_ccb);
                        xpt_free_ccb((union ccb *)scan_info->cpi);
                        request_ccb = scan_info->request_ccb;
                        free(scan_info, M_CAMXPT);
                        request_ccb->ccb_h.status = status;
                        xpt_done(request_ccb);
                        break;
                }
                xpt_setup_ccb(&work_ccb->ccb_h, path,
                    scan_info->request_ccb->ccb_h.pinfo.priority);
                work_ccb->ccb_h.func_code = XPT_SCAN_LUN;
                work_ccb->ccb_h.cbfcnp = ata_scan_bus;
                work_ccb->ccb_h.flags |= CAM_UNLOCKED;
                work_ccb->ccb_h.ppriv_ptr0 = scan_info;
                work_ccb->crcn.flags = scan_info->request_ccb->crcn.flags;
                mtx_unlock(mtx);
                if (request_ccb->ccb_h.func_code == XPT_SCAN_LUN)
                        mtx = NULL;
                xpt_action(work_ccb);
                if (mtx != NULL)
                        mtx_lock(mtx);
                break;
        default:
                break;
        }
}

static void
ata_scan_lun(struct cam_periph *periph, struct cam_path *path,
             cam_flags flags, union ccb *request_ccb)
{
        struct ccb_pathinq cpi;
        cam_status status;
        struct cam_path *new_path;
        struct cam_periph *old_periph;
        int lock;

        CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_scan_lun\n"));

        xpt_path_inq(&cpi, path);
        if (cpi.ccb_h.status != CAM_REQ_CMP) {
                if (request_ccb != NULL) {
                        request_ccb->ccb_h.status = cpi.ccb_h.status;
                        xpt_done(request_ccb);
                }
                return;
        }

        if (request_ccb == NULL) {
                request_ccb = xpt_alloc_ccb_nowait();
                if (request_ccb == NULL) {
                        xpt_print(path,
                            "xpt_scan_lun: can't allocate CCB, can't continue\n");
                        return;
                }
                status = xpt_create_path(&new_path, NULL,
                                          path->bus->path_id,
                                          path->target->target_id,
                                          path->device->lun_id);
                if (status != CAM_REQ_CMP) {
                        xpt_print(path,
                            "xpt_scan_lun: can't create path, can't continue\n");
                        xpt_free_ccb(request_ccb);
                        return;
                }
                xpt_setup_ccb(&request_ccb->ccb_h, new_path, CAM_PRIORITY_XPT);
                request_ccb->ccb_h.cbfcnp = axptscandone;
                request_ccb->ccb_h.flags |= CAM_UNLOCKED;
                request_ccb->ccb_h.func_code = XPT_SCAN_LUN;
                request_ccb->crcn.flags = flags;
        }

        lock = (xpt_path_owned(path) == 0);
        if (lock)
                xpt_path_lock(path);
        if ((old_periph = cam_periph_find(path, "aprobe")) != NULL) {
                if ((old_periph->flags & CAM_PERIPH_INVALID) == 0) {
                        aprobe_softc *softc;

                        softc = (aprobe_softc *)old_periph->softc;
                        TAILQ_INSERT_TAIL(&softc->request_ccbs,
                                &request_ccb->ccb_h, periph_links.tqe);
                        softc->restart = 1;
                } else {
                        request_ccb->ccb_h.status = CAM_REQ_CMP_ERR;
                        xpt_done(request_ccb);
                }
        } else {
                status = cam_periph_alloc(aproberegister, NULL, aprobecleanup,
                                          aprobestart, "aprobe",
                                          CAM_PERIPH_BIO,
                                          request_ccb->ccb_h.path, NULL, 0,
                                          request_ccb);

                if (status != CAM_REQ_CMP) {
                        xpt_print(path,
        "xpt_scan_lun: cam_alloc_periph returned an error, can't continue probe\n");
                        request_ccb->ccb_h.status = status;
                        xpt_done(request_ccb);
                }
        }
        if (lock)
                xpt_path_unlock(path);
}

static void
axptscandone(struct cam_periph *periph, union ccb *done_ccb)
{

        xpt_free_path(done_ccb->ccb_h.path);
        xpt_free_ccb(done_ccb);
}

static struct cam_ed *
ata_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
{
        struct ata_quirk_entry *quirk;
        struct cam_ed *device;

        device = xpt_alloc_device(bus, target, lun_id);
        if (device == NULL)
                return (NULL);

        /*
         * Take the default quirk entry until we have inquiry
         * data and can determine a better quirk to use.
         */
        quirk = &ata_quirk_table[nitems(ata_quirk_table) - 1];
        device->quirk = (void *)quirk;
        device->mintags = 0;
        device->maxtags = 0;
        bzero(&device->inq_data, sizeof(device->inq_data));
        device->inq_flags = 0;
        device->queue_flags = 0;
        device->serial_num = NULL;
        device->serial_num_len = 0;
        return (device);
}

static void
ata_device_transport(struct cam_path *path)
{
        struct ccb_pathinq cpi;
        struct ccb_trans_settings cts;
        struct scsi_inquiry_data *inq_buf = NULL;
        struct ata_params *ident_buf = NULL;

        /* Get transport information from the SIM */
        xpt_path_inq(&cpi, path);

        path->device->transport = cpi.transport;
        if ((path->device->flags & CAM_DEV_INQUIRY_DATA_VALID) != 0)
                inq_buf = &path->device->inq_data;
        if ((path->device->flags & CAM_DEV_IDENTIFY_DATA_VALID) != 0)
                ident_buf = &path->device->ident_data;
        if (path->device->protocol == PROTO_ATA) {
                path->device->protocol_version = ident_buf ?
                    ata_version(ident_buf->version_major) : cpi.protocol_version;
        } else if (path->device->protocol == PROTO_SCSI) {
                path->device->protocol_version = inq_buf ?
                    SID_ANSI_REV(inq_buf) : cpi.protocol_version;
        }
        path->device->transport_version = ident_buf ?
            ata_version(ident_buf->version_major) : cpi.transport_version;

        /* Tell the controller what we think */
        bzero(&cts, sizeof(cts));
        xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NONE);
        cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
        cts.type = CTS_TYPE_CURRENT_SETTINGS;
        cts.transport = path->device->transport;
        cts.transport_version = path->device->transport_version;
        cts.protocol = path->device->protocol;
        cts.protocol_version = path->device->protocol_version;
        cts.proto_specific.valid = 0;
        if (ident_buf) {
                if (path->device->transport == XPORT_ATA) {
                        cts.xport_specific.ata.atapi =
                            (ident_buf->config == ATA_PROTO_CFA) ? 0 :
                            ((ident_buf->config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_16) ? 16 :
                            ((ident_buf->config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_12) ? 12 : 0;
                        cts.xport_specific.ata.valid = CTS_ATA_VALID_ATAPI;
                } else {
                        cts.xport_specific.sata.atapi =
                            (ident_buf->config == ATA_PROTO_CFA) ? 0 :
                            ((ident_buf->config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_16) ? 16 :
                            ((ident_buf->config & ATA_PROTO_MASK) == ATA_PROTO_ATAPI_12) ? 12 : 0;
                        cts.xport_specific.sata.valid = CTS_SATA_VALID_ATAPI;
                }
        } else
                cts.xport_specific.valid = 0;
        xpt_action((union ccb *)&cts);
}

static void
ata_dev_advinfo(union ccb *start_ccb)
{
        struct cam_ed *device;
        struct ccb_dev_advinfo *cdai;
        off_t amt;

        xpt_path_assert(start_ccb->ccb_h.path, MA_OWNED);
        start_ccb->ccb_h.status = CAM_REQ_INVALID;
        device = start_ccb->ccb_h.path->device;
        cdai = &start_ccb->cdai;
        switch(cdai->buftype) {
        case CDAI_TYPE_SCSI_DEVID:
                if (cdai->flags & CDAI_FLAG_STORE)
                        return;
                cdai->provsiz = device->device_id_len;
                if (device->device_id_len == 0)
                        break;
                amt = device->device_id_len;
                if (cdai->provsiz > cdai->bufsiz)
                        amt = cdai->bufsiz;
                memcpy(cdai->buf, device->device_id, amt);
                break;
        case CDAI_TYPE_SERIAL_NUM:
                if (cdai->flags & CDAI_FLAG_STORE)
                        return;
                cdai->provsiz = device->serial_num_len;
                if (device->serial_num_len == 0)
                        break;
                amt = device->serial_num_len;
                if (cdai->provsiz > cdai->bufsiz)
                        amt = cdai->bufsiz;
                memcpy(cdai->buf, device->serial_num, amt);
                break;
        case CDAI_TYPE_PHYS_PATH:
                if (cdai->flags & CDAI_FLAG_STORE) {
                        if (device->physpath != NULL) {
                                free(device->physpath, M_CAMXPT);
                                device->physpath = NULL;
                                device->physpath_len = 0;
                        }
                        /* Clear existing buffer if zero length */
                        if (cdai->bufsiz == 0)
                                break;
                        device->physpath = malloc(cdai->bufsiz, M_CAMXPT, M_NOWAIT);
                        if (device->physpath == NULL) {
                                start_ccb->ccb_h.status = CAM_REQ_ABORTED;
                                return;
                        }
                        device->physpath_len = cdai->bufsiz;
                        memcpy(device->physpath, cdai->buf, cdai->bufsiz);
                } else {
                        cdai->provsiz = device->physpath_len;
                        if (device->physpath_len == 0)
                                break;
                        amt = device->physpath_len;
                        if (cdai->provsiz > cdai->bufsiz)
                                amt = cdai->bufsiz;
                        memcpy(cdai->buf, device->physpath, amt);
                }
                break;
        default:
                return;
        }
        start_ccb->ccb_h.status = CAM_REQ_CMP;

        if (cdai->flags & CDAI_FLAG_STORE) {
                xpt_async(AC_ADVINFO_CHANGED, start_ccb->ccb_h.path,
                          (void *)(uintptr_t)cdai->buftype);
        }
}

static void
ata_action(union ccb *start_ccb)
{

        if (start_ccb->ccb_h.func_code != XPT_ATA_IO) {
                KASSERT((start_ccb->ccb_h.alloc_flags & CAM_CCB_FROM_UMA) == 0,
                    ("%s: ccb %p, func_code %#x should not be allocated from UMA zone\n",
                    __func__, start_ccb, start_ccb->ccb_h.func_code));
        }

        switch (start_ccb->ccb_h.func_code) {
        case XPT_SET_TRAN_SETTINGS:
        {
                ata_set_transfer_settings(&start_ccb->cts,
                                           start_ccb->ccb_h.path,
                                           /*async_update*/FALSE);
                break;
        }
        case XPT_SCAN_BUS:
        case XPT_SCAN_TGT:
                ata_scan_bus(start_ccb->ccb_h.path->periph, start_ccb);
                break;
        case XPT_SCAN_LUN:
                ata_scan_lun(start_ccb->ccb_h.path->periph,
                              start_ccb->ccb_h.path, start_ccb->crcn.flags,
                              start_ccb);
                break;
        case XPT_GET_TRAN_SETTINGS:
        {
                ata_get_transfer_settings(&start_ccb->cts);
                break;
        }
        case XPT_SCSI_IO:
        {
                struct cam_ed *device;
                u_int   maxlen = 0;

                device = start_ccb->ccb_h.path->device;
                if (device->protocol == PROTO_SCSI &&
                    (device->flags & CAM_DEV_IDENTIFY_DATA_VALID)) {
                        uint16_t p =
                            device->ident_data.config & ATA_PROTO_MASK;

                        maxlen =
                            (device->ident_data.config == ATA_PROTO_CFA) ? 0 :
                            (p == ATA_PROTO_ATAPI_16) ? 16 :
                            (p == ATA_PROTO_ATAPI_12) ? 12 : 0;
                }
                if (start_ccb->csio.cdb_len > maxlen) {
                        start_ccb->ccb_h.status = CAM_REQ_INVALID;
                        xpt_done(start_ccb);
                        break;
                }
                xpt_action_default(start_ccb);
                break;
        }
        case XPT_DEV_ADVINFO:
        {
                ata_dev_advinfo(start_ccb);
                break;
        }
        default:
                xpt_action_default(start_ccb);
                break;
        }
}

static void
ata_get_transfer_settings(struct ccb_trans_settings *cts)
{
        struct  ccb_trans_settings_ata *ata;
        struct  ccb_trans_settings_scsi *scsi;
        struct  cam_ed *device;

        device = cts->ccb_h.path->device;
        xpt_action_default((union ccb *)cts);

        if (cts->protocol == PROTO_UNKNOWN ||
            cts->protocol == PROTO_UNSPECIFIED) {
                cts->protocol = device->protocol;
                cts->protocol_version = device->protocol_version;
        }

        if (cts->protocol == PROTO_ATA) {
                ata = &cts->proto_specific.ata;
                if ((ata->valid & CTS_ATA_VALID_TQ) == 0) {
                        ata->valid |= CTS_ATA_VALID_TQ;
                        if (cts->type == CTS_TYPE_USER_SETTINGS ||
                            (device->flags & CAM_DEV_TAG_AFTER_COUNT) != 0 ||
                            (device->inq_flags & SID_CmdQue) != 0)
                                ata->flags |= CTS_ATA_FLAGS_TAG_ENB;
                }
        }
        if (cts->protocol == PROTO_SCSI) {
                scsi = &cts->proto_specific.scsi;
                if ((scsi->valid & CTS_SCSI_VALID_TQ) == 0) {
                        scsi->valid |= CTS_SCSI_VALID_TQ;
                        if (cts->type == CTS_TYPE_USER_SETTINGS ||
                            (device->flags & CAM_DEV_TAG_AFTER_COUNT) != 0 ||
                            (device->inq_flags & SID_CmdQue) != 0)
                                scsi->flags |= CTS_SCSI_FLAGS_TAG_ENB;
                }
        }

        if (cts->transport == XPORT_UNKNOWN ||
            cts->transport == XPORT_UNSPECIFIED) {
                cts->transport = device->transport;
                cts->transport_version = device->transport_version;
        }
}

static void
ata_set_transfer_settings(struct ccb_trans_settings *cts, struct cam_path *path,
                           int async_update)
{
        struct  ccb_pathinq cpi;
        struct  ccb_trans_settings_ata *ata;
        struct  ccb_trans_settings_scsi *scsi;
        struct  ata_params *ident_data;
        struct  scsi_inquiry_data *inq_data;
        struct  cam_ed *device;

        if (path == NULL || (device = path->device) == NULL) {
                cts->ccb_h.status = CAM_PATH_INVALID;
                xpt_done((union ccb *)cts);
                return;
        }

        if (cts->protocol == PROTO_UNKNOWN
         || cts->protocol == PROTO_UNSPECIFIED) {
                cts->protocol = device->protocol;
                cts->protocol_version = device->protocol_version;
        }

        if (cts->protocol_version == PROTO_VERSION_UNKNOWN
         || cts->protocol_version == PROTO_VERSION_UNSPECIFIED)
                cts->protocol_version = device->protocol_version;

        if (cts->protocol != device->protocol) {
                xpt_print(path, "Uninitialized Protocol %x:%x?\n",
                       cts->protocol, device->protocol);
                cts->protocol = device->protocol;
        }

        if (cts->protocol_version > device->protocol_version) {
                if (bootverbose) {
                        xpt_print(path,
                            "Down reving Protocol Version from %d to %d?\n",
                            cts->protocol_version, device->protocol_version);
                }
                cts->protocol_version = device->protocol_version;
        }

        if (cts->transport == XPORT_UNKNOWN
         || cts->transport == XPORT_UNSPECIFIED) {
                cts->transport = device->transport;
                cts->transport_version = device->transport_version;
        }

        if (cts->transport_version == XPORT_VERSION_UNKNOWN
         || cts->transport_version == XPORT_VERSION_UNSPECIFIED)
                cts->transport_version = device->transport_version;

        if (cts->transport != device->transport) {
                xpt_print(path, "Uninitialized Transport %x:%x?\n",
                    cts->transport, device->transport);
                cts->transport = device->transport;
        }

        if (cts->transport_version > device->transport_version) {
                if (bootverbose) {
                        xpt_print(path,
                            "Down reving Transport Version from %d to %d?\n",
                            cts->transport_version, device->transport_version);
                }
                cts->transport_version = device->transport_version;
        }

        ident_data = &device->ident_data;
        inq_data = &device->inq_data;
        if (cts->protocol == PROTO_ATA)
                ata = &cts->proto_specific.ata;
        else
                ata = NULL;
        if (cts->protocol == PROTO_SCSI)
                scsi = &cts->proto_specific.scsi;
        else
                scsi = NULL;
        xpt_path_inq(&cpi, path);

        /* Sanity checking */
        if ((cpi.hba_inquiry & PI_TAG_ABLE) == 0
         || (ata && (ident_data->satacapabilities & ATA_SUPPORT_NCQ) == 0)
         || (scsi && (INQ_DATA_TQ_ENABLED(inq_data)) == 0)
         || (device->queue_flags & SCP_QUEUE_DQUE) != 0
         || (device->mintags == 0)) {
                /*
                 * Can't tag on hardware that doesn't support tags,
                 * doesn't have it enabled, or has broken tag support.
                 */
                if (ata)
                        ata->flags &= ~CTS_ATA_FLAGS_TAG_ENB;
                if (scsi)
                        scsi->flags &= ~CTS_SCSI_FLAGS_TAG_ENB;
        }

        /* Start/stop tags use. */
        if (cts->type == CTS_TYPE_CURRENT_SETTINGS &&
            ((ata && (ata->valid & CTS_ATA_VALID_TQ) != 0) ||
             (scsi && (scsi->valid & CTS_SCSI_VALID_TQ) != 0))) {
                int nowt, newt = 0;

                nowt = ((device->flags & CAM_DEV_TAG_AFTER_COUNT) != 0 ||
                        (device->inq_flags & SID_CmdQue) != 0);
                if (ata)
                        newt = (ata->flags & CTS_ATA_FLAGS_TAG_ENB) != 0;
                if (scsi)
                        newt = (scsi->flags & CTS_SCSI_FLAGS_TAG_ENB) != 0;

                if (newt && !nowt) {
                        /*
                         * Delay change to use tags until after a
                         * few commands have gone to this device so
                         * the controller has time to perform transfer
                         * negotiations without tagged messages getting
                         * in the way.
                         */
                        device->tag_delay_count = CAM_TAG_DELAY_COUNT;
                        device->flags |= CAM_DEV_TAG_AFTER_COUNT;
                } else if (nowt && !newt)
                        xpt_stop_tags(path);
        }

        if (async_update == FALSE)
                xpt_action_default((union ccb *)cts);
}

/*
 * Handle any per-device event notifications that require action by the XPT.
 */
static void
ata_dev_async(uint32_t async_code, struct cam_eb *bus, struct cam_et *target,
              struct cam_ed *device, void *async_arg)
{
        /*
         * We only need to handle events for real devices.
         */
        if (target->target_id == CAM_TARGET_WILDCARD ||
            device->lun_id == CAM_LUN_WILDCARD)
                return;

        switch (async_code) {
        case AC_SENT_BDR:
        case AC_BUS_RESET:
        case AC_INQ_CHANGED: {
                cam_status status;
                struct cam_path newpath;
                cam_flags flags;

                /*
                 * We need our own path with wildcards expanded to handle these
                 * events.
                 */
                status = xpt_compile_path(&newpath, NULL,
                                          bus->path_id,
                                          target->target_id,
                                          device->lun_id);
                if (status != CAM_REQ_CMP)
                        break; /* fail safe and just drop it */

                /*
                 * For AC_INQ_CHANGED, we've sent a start unit command, or
                 * something similar to a device that may have caused its
                 * inquiry data to change. So we re-scan the device to refresh
                 * the inquiry data for it, allowing changes. Otherwise we rescan
                 * without allowing changes to respond to the reset, not allowing
                 * changes.
                 */
                flags = async_code == AC_INQ_CHANGED ? CAM_EXPECT_INQ_CHANGE : 0;
                ata_scan_lun(newpath.periph, &newpath, flags, NULL);
                xpt_release_path(&newpath);
                break;
        }
        case AC_TRANSFER_NEG: {
                struct ccb_trans_settings *settings;
                struct cam_path path;

                settings = (struct ccb_trans_settings *)async_arg;
                xpt_compile_path(&path, NULL, bus->path_id, target->target_id,
                                 device->lun_id);
                ata_set_transfer_settings(settings, &path,
                                          /*async_update*/TRUE);
                xpt_release_path(&path);
                break;
        }
        case AC_LOST_DEVICE:
                if ((device->flags & CAM_DEV_UNCONFIGURED) == 0) {
                        device->flags |= CAM_DEV_UNCONFIGURED;
                        xpt_release_device(device);
                }
                break;
        }
}

static void
_ata_announce_periph(struct cam_periph *periph, struct ccb_trans_settings *cts, u_int *speed)
{
        struct  ccb_pathinq cpi;
        struct  cam_path *path = periph->path;

        cam_periph_assert(periph, MA_OWNED);

        xpt_setup_ccb(&cts->ccb_h, path, CAM_PRIORITY_NORMAL);
        cts->ccb_h.func_code = XPT_GET_TRAN_SETTINGS;
        cts->type = CTS_TYPE_CURRENT_SETTINGS;
        xpt_action((union ccb*)cts);
        if ((cts->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
                return;
        /* Ask the SIM for its base transfer speed */
        xpt_path_inq(&cpi, path);
        /* Report connection speed */
        *speed = cpi.base_transfer_speed;
        if (cts->transport == XPORT_ATA) {
                struct  ccb_trans_settings_pata *pata =
                    &cts->xport_specific.ata;

                if (pata->valid & CTS_ATA_VALID_MODE)
                        *speed = ata_mode2speed(pata->mode);
        }
        if (cts->transport == XPORT_SATA) {
                struct  ccb_trans_settings_sata *sata =
                    &cts->xport_specific.sata;

                if (sata->valid & CTS_SATA_VALID_REVISION)
                        *speed = ata_revision2speed(sata->revision);
        }
}

static void
ata_announce_periph_sbuf(struct cam_periph *periph, struct sbuf *sb)
{
        struct ccb_trans_settings cts;
        u_int speed, mb;

        bzero(&cts, sizeof(cts));
        _ata_announce_periph(periph, &cts, &speed);
        if ((cts.ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
                return;

        mb = speed / 1000;
        if (mb > 0)
                sbuf_printf(sb, "%s%d: %d.%03dMB/s transfers",
                       periph->periph_name, periph->unit_number,
                       mb, speed % 1000);
        else
                sbuf_printf(sb, "%s%d: %dKB/s transfers", periph->periph_name,
                       periph->unit_number, speed);
        /* Report additional information about connection */
        if (cts.transport == XPORT_ATA) {
                struct ccb_trans_settings_pata *pata =
                    &cts.xport_specific.ata;

                sbuf_cat(sb, " (");
                if (pata->valid & CTS_ATA_VALID_MODE)
                        sbuf_printf(sb, "%s, ", ata_mode2string(pata->mode));
                if ((pata->valid & CTS_ATA_VALID_ATAPI) && pata->atapi != 0)
                        sbuf_printf(sb, "ATAPI %dbytes, ", pata->atapi);
                if (pata->valid & CTS_ATA_VALID_BYTECOUNT)
                        sbuf_printf(sb, "PIO %dbytes", pata->bytecount);
                sbuf_putc(sb, ')');
        }
        if (cts.transport == XPORT_SATA) {
                struct ccb_trans_settings_sata *sata =
                    &cts.xport_specific.sata;

                sbuf_cat(sb, " (");
                if (sata->valid & CTS_SATA_VALID_REVISION)
                        sbuf_printf(sb, "SATA %d.x, ", sata->revision);
                else
                        sbuf_cat(sb, "SATA, ");
                if (sata->valid & CTS_SATA_VALID_MODE)
                        sbuf_printf(sb, "%s, ", ata_mode2string(sata->mode));
                if ((sata->valid & CTS_ATA_VALID_ATAPI) && sata->atapi != 0)
                        sbuf_printf(sb, "ATAPI %dbytes, ", sata->atapi);
                if (sata->valid & CTS_SATA_VALID_BYTECOUNT)
                        sbuf_printf(sb, "PIO %dbytes", sata->bytecount);
                sbuf_putc(sb, ')');
        }
        sbuf_putc(sb, '\n');
}

static void
ata_proto_announce_sbuf(struct cam_ed *device, struct sbuf *sb)
{
        ata_print_ident_sbuf(&device->ident_data, sb);
}

static void
ata_proto_denounce_sbuf(struct cam_ed *device, struct sbuf *sb)
{
        ata_print_ident_short_sbuf(&device->ident_data, sb);
}

static void
semb_proto_announce_sbuf(struct cam_ed *device, struct sbuf *sb)
{
        semb_print_ident_sbuf((struct sep_identify_data *)&device->ident_data, sb);
}

static void
semb_proto_denounce_sbuf(struct cam_ed *device, struct sbuf *sb)
{
        semb_print_ident_short_sbuf((struct sep_identify_data *)&device->ident_data, sb);
}

static void
ata_proto_debug_out(union ccb *ccb)
{
        char cdb_str[(sizeof(struct ata_cmd) * 3) + 1];

        if (ccb->ccb_h.func_code != XPT_ATA_IO)
                return;

        CAM_DEBUG(ccb->ccb_h.path,
            CAM_DEBUG_CDB,("%s. ACB: %s\n", ata_op_string(&ccb->ataio.cmd),
                ata_cmd_string(&ccb->ataio.cmd, cdb_str, sizeof(cdb_str))));
}