/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
 * Copyright 2018 Nexenta Systems, Inc.  All rights reserved.
 * Copyright (c) 2018, Joyent, Inc.
 * Copyright 2018 OmniOS Community Edition (OmniOSce) Association.
 * Copyright 2021 RackTop Systems, Inc.
 */

/*
 * AHCI (Advanced Host Controller Interface) SATA HBA Driver
 *
 * Power Management Support
 * ------------------------
 *
 * At the moment, the ahci driver only implements suspend/resume to
 * support Suspend to RAM on X86 feature. Device power management isn't
 * implemented, link power management is disabled, and hot plug isn't
 * allowed during the period from suspend to resume.
 *
 * For s/r support, the ahci driver only need to implement DDI_SUSPEND
 * and DDI_RESUME entries, and don't need to take care of new requests
 * sent down after suspend because the target driver (sd) has already
 * handled these conditions, and blocked these requests. For the detailed
 * information, please check with sdopen, sdclose and sdioctl routines.
 *
 *
 * Enclosure Management Support
 * ----------------------------
 *
 * The ahci driver has basic support for AHCI Enclosure Management (EM)
 * services. The AHCI specification provides an area in the primary ahci BAR for
 * posting data to send out to the enclosure management and provides a register
 * that provides both information and control about this. While the
 * specification allows for multiple forms of enclosure management, the only
 * supported, and commonly found form, is the AHCI specified LED format. The LED
 * format is often implemented as a one-way communication mechanism. Software
 * can write out what it cares about into the aforementioned data buffer and
 * then we wait for the transmission to be sent.
 *
 * This has some drawbacks. It means that we cannot know whether or not it has
 * succeeded. This means we cannot ask hardware what it thinks the LEDs are
 * set to. There's also the added unfortunate reality that firmware on the
 * microcontroller driving this will often not show the LEDs if no drive is
 * present and that actions taken may potentially cause this to get out of sync
 * with what we expect it to be. For example, the specification does not
 * describe what should happen if a drive is removed from the enclosure while
 * this is set and what should happen when it returns. We can only infer that it
 * should be the same.
 *
 * Because only a single command can be sent at any time and we don't want to
 * interfere with controller I/O, we create a taskq dedicated to this that has a
 * single thread. Both resets (which occur on attach and resume) and normal
 * changes to the LED state will be driven through this taskq. Because the taskq
 * has a single thread, this guarantees serial processing.
 *
 * Each userland-submitted task (basically not resets) has a reference counted
 * task structure. This allows the thread that called it to be cancelled and
 * have the system clean itself up. The user thread in ioctl blocks on a CV that
 * can receive signals as it waits for completion.  Note, there is no guarantee
 * provided by the kernel that the first thread to enter the kernel will be the
 * first one to change state.
 */

#include <sys/note.h>
#include <sys/scsi/scsi.h>
#include <sys/pci.h>
#include <sys/disp.h>
#include <sys/sata/sata_hba.h>
#include <sys/sata/adapters/ahci/ahcireg.h>
#include <sys/sata/adapters/ahci/ahcivar.h>

/*
 * FMA header files
 */
#include <sys/ddifm.h>
#include <sys/fm/protocol.h>
#include <sys/fm/util.h>
#include <sys/fm/io/ddi.h>

/*
 * EM Control header files
 */
#include <sys/types.h>
#include <sys/file.h>
#include <sys/errno.h>
#include <sys/open.h>
#include <sys/cred.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>

/*
 * This is the string displayed by modinfo, etc.
 */
static char ahci_ident[] = "ahci driver";

/*
 * Function prototypes for driver entry points
 */
static  int ahci_attach(dev_info_t *, ddi_attach_cmd_t);
static  int ahci_detach(dev_info_t *, ddi_detach_cmd_t);
static  int ahci_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
static  int ahci_quiesce(dev_info_t *);

/*
 * Function prototypes for SATA Framework interfaces
 */
static  int ahci_register_sata_hba_tran(ahci_ctl_t *, uint32_t);
static  int ahci_unregister_sata_hba_tran(ahci_ctl_t *);

static  int ahci_tran_probe_port(dev_info_t *, sata_device_t *);
static  int ahci_tran_start(dev_info_t *, sata_pkt_t *spkt);
static  int ahci_tran_abort(dev_info_t *, sata_pkt_t *, int);
static  int ahci_tran_reset_dport(dev_info_t *, sata_device_t *);
static  int ahci_tran_hotplug_port_activate(dev_info_t *, sata_device_t *);
static  int ahci_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *);
#if defined(__lock_lint)
static  int ahci_selftest(dev_info_t *, sata_device_t *);
#endif

/*
 * FMA Prototypes
 */
static  void ahci_fm_init(ahci_ctl_t *);
static  void ahci_fm_fini(ahci_ctl_t *);
static  int ahci_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void*);
int     ahci_check_acc_handle(ddi_acc_handle_t);
int     ahci_check_dma_handle(ddi_dma_handle_t);
void    ahci_fm_ereport(ahci_ctl_t *, char *);
static  int ahci_check_all_handle(ahci_ctl_t *);
static  int ahci_check_ctl_handle(ahci_ctl_t *);
static  int ahci_check_port_handle(ahci_ctl_t *, int);
static  int ahci_check_slot_handle(ahci_port_t *, int);

/*
 * Local function prototypes
 */
static  int ahci_setup_port_base_addresses(ahci_ctl_t *, ahci_port_t *);
static  int ahci_alloc_ports_state(ahci_ctl_t *);
static  void ahci_dealloc_ports_state(ahci_ctl_t *);
static  int ahci_alloc_port_state(ahci_ctl_t *, uint8_t);
static  void ahci_dealloc_port_state(ahci_ctl_t *, uint8_t);
static  int ahci_alloc_rcvd_fis(ahci_ctl_t *, ahci_port_t *);
static  void ahci_dealloc_rcvd_fis(ahci_port_t *);
static  int ahci_alloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
static  void ahci_dealloc_cmd_list(ahci_ctl_t *, ahci_port_t *);
static  int ahci_alloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
static  void ahci_dealloc_cmd_tables(ahci_ctl_t *, ahci_port_t *);
static  void ahci_alloc_pmult(ahci_ctl_t *, ahci_port_t *);
static  void ahci_dealloc_pmult(ahci_ctl_t *, ahci_port_t *);

static  int ahci_initialize_controller(ahci_ctl_t *);
static  void ahci_uninitialize_controller(ahci_ctl_t *);
static  int ahci_initialize_port(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
static  int ahci_config_space_init(ahci_ctl_t *);
static  void ahci_staggered_spin_up(ahci_ctl_t *, uint8_t);

static  void ahci_drain_ports_taskq(ahci_ctl_t *);
static  int ahci_rdwr_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *,
    uint8_t);
static  int ahci_read_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t *);
static  int ahci_write_pmult(ahci_ctl_t *, ahci_addr_t *, uint8_t, uint32_t);
static  int ahci_update_pmult_pscr(ahci_ctl_t *, ahci_addr_t *,
    sata_device_t *);
static  int ahci_update_pmult_gscr(ahci_ctl_t *, ahci_addr_t *,
    sata_pmult_gscr_t *);
static  int ahci_initialize_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
    sata_device_t *);
static  int ahci_initialize_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
static  int ahci_probe_pmult(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
static  int ahci_probe_pmport(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *,
    sata_device_t *);

static  void ahci_disable_interface_pm(ahci_ctl_t *, uint8_t);
static  int ahci_start_port(ahci_ctl_t *, ahci_port_t *, uint8_t);
static  void ahci_find_dev_signature(ahci_ctl_t *, ahci_port_t *,
    ahci_addr_t *);
static  void ahci_update_sata_registers(ahci_ctl_t *, uint8_t, sata_device_t *);
static  int ahci_deliver_satapkt(ahci_ctl_t *, ahci_port_t *,
    ahci_addr_t *, sata_pkt_t *);
static  int ahci_do_sync_start(ahci_ctl_t *, ahci_port_t *,
    ahci_addr_t *, sata_pkt_t *);
static  int ahci_claim_free_slot(ahci_ctl_t *, ahci_port_t *,
    ahci_addr_t *, int);
static  void ahci_copy_err_cnxt(sata_cmd_t *, ahci_fis_d2h_register_t *);
static  void ahci_copy_ncq_err_page(sata_cmd_t *,
    struct sata_ncq_error_recovery_page *);
static  void ahci_copy_out_regs(sata_cmd_t *, ahci_fis_d2h_register_t *);
static  void ahci_add_doneq(ahci_port_t *, sata_pkt_t *, int);
static  void ahci_flush_doneq(ahci_port_t *);

static  int ahci_software_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
static  int ahci_hba_reset(ahci_ctl_t *);
static  int ahci_port_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
static  int ahci_pmport_reset(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
static  void ahci_reject_all_abort_pkts(ahci_ctl_t *, ahci_port_t *, uint8_t);
static  int ahci_reset_device_reject_pkts(ahci_ctl_t *, ahci_port_t *,
    ahci_addr_t *);
static  int ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *, ahci_port_t *,
    ahci_addr_t *);
static  int ahci_reset_port_reject_pkts(ahci_ctl_t *, ahci_port_t *,
    ahci_addr_t *);
static  int ahci_reset_hba_reject_pkts(ahci_ctl_t *);
static  int ahci_put_port_into_notrunning_state(ahci_ctl_t *, ahci_port_t *,
    uint8_t);
static  int ahci_restart_port_wait_till_ready(ahci_ctl_t *, ahci_port_t *,
    uint8_t, int, int *);
static  void ahci_mop_commands(ahci_ctl_t *, ahci_port_t *, uint32_t,
    uint32_t, uint32_t, uint32_t, uint32_t);
static  uint32_t ahci_get_rdlogext_data(ahci_ctl_t *, ahci_port_t *, uint8_t);
static void ahci_get_rqsense_data(ahci_ctl_t *, ahci_port_t *,
    uint8_t, sata_pkt_t *);
static  void ahci_fatal_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
    ahci_addr_t *, uint32_t);
static  void ahci_pmult_error_recovery_handler(ahci_ctl_t *, ahci_port_t *,
    uint8_t, uint32_t);
static  void ahci_timeout_pkts(ahci_ctl_t *, ahci_port_t *,
    uint8_t, uint32_t);
static  void ahci_events_handler(void *);
static  void ahci_watchdog_handler(ahci_ctl_t *);

static  uint_t ahci_intr(caddr_t, caddr_t);
static  void ahci_port_intr(ahci_ctl_t *, ahci_port_t *, uint8_t);
static  int ahci_add_intrs(ahci_ctl_t *, int);
static  void ahci_rem_intrs(ahci_ctl_t *);
static  void ahci_enable_all_intrs(ahci_ctl_t *);
static  void ahci_disable_all_intrs(ahci_ctl_t *);
static  void ahci_enable_port_intrs(ahci_ctl_t *, uint8_t);
static  void ahci_disable_port_intrs(ahci_ctl_t *, uint8_t);

static  int ahci_intr_cmd_cmplt(ahci_ctl_t *, ahci_port_t *, uint8_t);
static  int ahci_intr_set_device_bits(ahci_ctl_t *, ahci_port_t *, uint8_t);
static  int ahci_intr_ncq_events(ahci_ctl_t *, ahci_port_t *, ahci_addr_t *);
static  int ahci_intr_pmult_sntf_events(ahci_ctl_t *, ahci_port_t *, uint8_t);
static  int ahci_intr_port_connect_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
static  int ahci_intr_device_mechanical_presence_status(ahci_ctl_t *,
    ahci_port_t *, uint8_t);
static  int ahci_intr_phyrdy_change(ahci_ctl_t *, ahci_port_t *, uint8_t);
static  int ahci_intr_non_fatal_error(ahci_ctl_t *, ahci_port_t *,
    uint8_t, uint32_t);
static  int ahci_intr_fatal_error(ahci_ctl_t *, ahci_port_t *,
    uint8_t, uint32_t);
static  int ahci_intr_cold_port_detect(ahci_ctl_t *, ahci_port_t *, uint8_t);

static  void ahci_get_ahci_addr(ahci_ctl_t *, sata_device_t *, ahci_addr_t *);
static  int ahci_get_num_implemented_ports(uint32_t);
static  void ahci_log_fatal_error_message(ahci_ctl_t *, uint8_t, uint32_t);
static  void ahci_dump_commands(ahci_ctl_t *, uint8_t, uint32_t);
static  void ahci_log_serror_message(ahci_ctl_t *, uint8_t, uint32_t, int);
#if AHCI_DEBUG
static  void ahci_log(ahci_ctl_t *, uint_t, char *, ...);
#endif

static  boolean_t ahci_em_init(ahci_ctl_t *);
static  void ahci_em_fini(ahci_ctl_t *);
static  void ahci_em_suspend(ahci_ctl_t *);
static  void ahci_em_resume(ahci_ctl_t *);
static  int ahci_em_ioctl(dev_info_t *, int, intptr_t);


/*
 * DMA attributes for the data buffer
 *
 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 * does not support 64-bit addressing
 */
static ddi_dma_attr_t buffer_dma_attr = {
        DMA_ATTR_V0,            /* dma_attr_version */
        0x0ull,                 /* dma_attr_addr_lo: lowest bus address */
        0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
        0x3fffffull,            /* dma_attr_count_max i.e. for one cookie */
        0x2ull,                 /* dma_attr_align: word aligned */
        1,                      /* dma_attr_burstsizes */
        1,                      /* dma_attr_minxfer */
        0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
        0xffffffffull,          /* dma_attr_seg */
        AHCI_PRDT_NUMBER,       /* dma_attr_sgllen */
        512,                    /* dma_attr_granular */
        0,                      /* dma_attr_flags */
};

/*
 * DMA attributes for the rcvd FIS
 *
 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 * does not support 64-bit addressing
 */
static ddi_dma_attr_t rcvd_fis_dma_attr = {
        DMA_ATTR_V0,            /* dma_attr_version */
        0x0ull,                 /* dma_attr_addr_lo: lowest bus address */
        0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
        0xffffffffull,          /* dma_attr_count_max i.e. for one cookie */
        0x100ull,               /* dma_attr_align: 256-byte aligned */
        1,                      /* dma_attr_burstsizes */
        1,                      /* dma_attr_minxfer */
        0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
        0xffffffffull,          /* dma_attr_seg */
        1,                      /* dma_attr_sgllen */
        1,                      /* dma_attr_granular */
        0,                      /* dma_attr_flags */
};

/*
 * DMA attributes for the command list
 *
 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 * does not support 64-bit addressing
 */
static ddi_dma_attr_t cmd_list_dma_attr = {
        DMA_ATTR_V0,            /* dma_attr_version */
        0x0ull,                 /* dma_attr_addr_lo: lowest bus address */
        0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
        0xffffffffull,          /* dma_attr_count_max i.e. for one cookie */
        0x400ull,               /* dma_attr_align: 1K-byte aligned */
        1,                      /* dma_attr_burstsizes */
        1,                      /* dma_attr_minxfer */
        0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
        0xffffffffull,          /* dma_attr_seg */
        1,                      /* dma_attr_sgllen */
        1,                      /* dma_attr_granular */
        0,                      /* dma_attr_flags */
};

/*
 * DMA attributes for cmd tables
 *
 * dma_attr_addr_hi will be changed to 0xffffffffull if the HBA
 * does not support 64-bit addressing
 */
static ddi_dma_attr_t cmd_table_dma_attr = {
        DMA_ATTR_V0,            /* dma_attr_version */
        0x0ull,                 /* dma_attr_addr_lo: lowest bus address */
        0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
        0xffffffffull,          /* dma_attr_count_max i.e. for one cookie */
        0x80ull,                /* dma_attr_align: 128-byte aligned */
        1,                      /* dma_attr_burstsizes */
        1,                      /* dma_attr_minxfer */
        0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
        0xffffffffull,          /* dma_attr_seg */
        1,                      /* dma_attr_sgllen */
        1,                      /* dma_attr_granular */
        0,                      /* dma_attr_flags */
};


/* Device access attributes */
static ddi_device_acc_attr_t accattr = {
        DDI_DEVICE_ATTR_V1,
        DDI_STRUCTURE_LE_ACC,
        DDI_STRICTORDER_ACC,
        DDI_DEFAULT_ACC
};

static struct dev_ops ahcictl_dev_ops = {
        DEVO_REV,               /* devo_rev */
        0,                      /* refcnt  */
        ahci_getinfo,           /* info */
        nulldev,                /* identify */
        nulldev,                /* probe */
        ahci_attach,            /* attach */
        ahci_detach,            /* detach */
        nodev,                  /* no reset */
        NULL,                   /* driver operations */
        NULL,                   /* bus operations */
        NULL,                   /* power */
        ahci_quiesce,           /* quiesce */
};

static sata_tran_hotplug_ops_t ahci_tran_hotplug_ops = {
        SATA_TRAN_HOTPLUG_OPS_REV_1,
        ahci_tran_hotplug_port_activate,
        ahci_tran_hotplug_port_deactivate
};

extern struct mod_ops mod_driverops;

static  struct modldrv modldrv = {
        &mod_driverops,         /* driverops */
        ahci_ident,             /* short description */
        &ahcictl_dev_ops,       /* driver ops */
};

static  struct modlinkage modlinkage = {
        MODREV_1,
        &modldrv,
        NULL
};

/* The following variables are watchdog handler related */
static clock_t ahci_watchdog_timeout = 5; /* 5 seconds */
static clock_t ahci_watchdog_tick;

/*
 * This static variable indicates the size of command table,
 * and it's changeable with prdt number, which ahci_dma_prdt_number
 * indicates.
 */
static size_t ahci_cmd_table_size;

/*
 * The below global variables are tunable via /etc/system
 *
 *      ahci_dma_prdt_number
 *      ahci_msi_enabled
 *      ahci_buf_64bit_dma
 *      ahci_commu_64bit_dma
 */

/* The number of Physical Region Descriptor Table(PRDT) in Command Table */
int ahci_dma_prdt_number = AHCI_PRDT_NUMBER;

/* AHCI MSI is tunable */
boolean_t ahci_msi_enabled = B_TRUE;

/*
 * 64-bit dma addressing for data buffer is tunable
 *
 * The variable controls only the below value:
 *      DBAU (upper 32-bits physical address of data block)
 */
boolean_t ahci_buf_64bit_dma = B_TRUE;

/*
 * 64-bit dma addressing for communication system descriptors is tunable
 *
 * The variable controls the below three values:
 *
 *      PxCLBU (upper 32-bits for the command list base physical address)
 *      PxFBU (upper 32-bits for the received FIS base physical address)
 *      CTBAU (upper 32-bits of command table base)
 */
boolean_t ahci_commu_64bit_dma = B_TRUE;

/*
 * By default, 64-bit dma for data buffer will be disabled for AMD/ATI SB600
 * chipset. If the users want to have a try with 64-bit dma, please change
 * the below variable value to enable it.
 */
boolean_t sb600_buf_64bit_dma_disable = B_TRUE;

/*
 * By default, 64-bit dma for command buffer will be disabled for AMD/ATI
 * SB600/700/710/750/800. If the users want to have a try with 64-bit dma,
 * please change the below value to enable it.
 */
boolean_t sbxxx_commu_64bit_dma_disable = B_TRUE;

/*
 * These values control the default delay and default number of times to wait
 * for an enclosure message to complete.
 */
uint_t  ahci_em_reset_delay_ms = 1;
uint_t  ahci_em_reset_delay_count = 1000;
uint_t  ahci_em_tx_delay_ms = 1;
uint_t  ahci_em_tx_delay_count = 1000;


/*
 * End of global tunable variable definition
 */

#if AHCI_DEBUG
uint32_t ahci_debug_flags = 0;
#else
uint32_t ahci_debug_flags = (AHCIDBG_ERRS|AHCIDBG_TIMEOUT);
#endif


#if AHCI_DEBUG
/* The following is needed for ahci_log() */
static kmutex_t ahci_log_mutex;
static char ahci_log_buf[512];
#endif

/* Opaque state pointer initialized by ddi_soft_state_init() */
static void *ahci_statep = NULL;

/*
 *  ahci module initialization.
 */
int
_init(void)
{
        int     ret;

        ret = ddi_soft_state_init(&ahci_statep, sizeof (ahci_ctl_t), 0);
        if (ret != 0) {
                goto err_out;
        }

#if AHCI_DEBUG
        mutex_init(&ahci_log_mutex, NULL, MUTEX_DRIVER, NULL);
#endif

        if ((ret = sata_hba_init(&modlinkage)) != 0) {
#if AHCI_DEBUG
                mutex_destroy(&ahci_log_mutex);
#endif
                ddi_soft_state_fini(&ahci_statep);
                goto err_out;
        }

        /* watchdog tick */
        ahci_watchdog_tick = drv_usectohz(
            (clock_t)ahci_watchdog_timeout * 1000000);

        ret = mod_install(&modlinkage);
        if (ret != 0) {
                sata_hba_fini(&modlinkage);
#if AHCI_DEBUG
                mutex_destroy(&ahci_log_mutex);
#endif
                ddi_soft_state_fini(&ahci_statep);
                goto err_out;
        }

        return (ret);

err_out:
        cmn_err(CE_WARN, "!ahci: Module init failed");
        return (ret);
}

/*
 * ahci module uninitialize.
 */
int
_fini(void)
{
        int     ret;

        ret = mod_remove(&modlinkage);
        if (ret != 0) {
                return (ret);
        }

        /* Remove the resources allocated in _init(). */
        sata_hba_fini(&modlinkage);
#if AHCI_DEBUG
        mutex_destroy(&ahci_log_mutex);
#endif
        ddi_soft_state_fini(&ahci_statep);

        return (ret);
}

/*
 * _info entry point
 */
int
_info(struct modinfo *modinfop)
{
        return (mod_info(&modlinkage, modinfop));
}

/*
 * The attach entry point for dev_ops.
 */
static int
ahci_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
        ahci_ctl_t *ahci_ctlp = NULL;
        int instance = ddi_get_instance(dip);
        int status;
        int attach_state;
        uint32_t cap_status, ahci_version;
        uint32_t ghc_control;
        int intr_types;
        int i;
        pci_regspec_t *regs;
        int regs_length;
        int rnumber;
#if AHCI_DEBUG
        int speed;
#endif

        AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "ahci_attach enter",
            NULL);

        switch (cmd) {
        case DDI_ATTACH:
                break;

        case DDI_RESUME:

                /*
                 * During DDI_RESUME, the hardware state of the device
                 * (power may have been removed from the device) must be
                 * restored, allow pending requests to continue, and
                 * service new requests.
                 */
                ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
                mutex_enter(&ahci_ctlp->ahcictl_mutex);

                /*
                 * GHC.AE must be set to 1 before any other AHCI register
                 * is accessed
                 */
                ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
                ghc_control |= AHCI_HBA_GHC_AE;
                ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);

                /* Restart watch thread */
                if (ahci_ctlp->ahcictl_timeout_id == 0)
                        ahci_ctlp->ahcictl_timeout_id = timeout(
                            (void (*)(void *))ahci_watchdog_handler,
                            (caddr_t)ahci_ctlp, ahci_watchdog_tick);

                mutex_exit(&ahci_ctlp->ahcictl_mutex);

                /*
                 * Re-initialize the controller and enable the interrupts and
                 * restart all the ports.
                 *
                 * Note that so far we don't support hot-plug during
                 * suspend/resume.
                 */
                if (ahci_initialize_controller(ahci_ctlp) != AHCI_SUCCESS) {
                        AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PM, ahci_ctlp,
                            "Failed to initialize the controller "
                            "during DDI_RESUME", NULL);
                        return (DDI_FAILURE);
                }

                /*
                 * Reset the enclosure services.
                 */
                ahci_em_resume(ahci_ctlp);

                mutex_enter(&ahci_ctlp->ahcictl_mutex);
                ahci_ctlp->ahcictl_flags &= ~AHCI_SUSPEND;
                mutex_exit(&ahci_ctlp->ahcictl_mutex);

                return (DDI_SUCCESS);

        default:
                return (DDI_FAILURE);
        }

        attach_state = AHCI_ATTACH_STATE_NONE;

        /* Allocate soft state */
        status = ddi_soft_state_zalloc(ahci_statep, instance);
        if (status != DDI_SUCCESS) {
                cmn_err(CE_WARN, "!ahci%d: Cannot allocate soft state",
                    instance);
                goto err_out;
        }

        ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
        ahci_ctlp->ahcictl_flags |= AHCI_ATTACH;
        ahci_ctlp->ahcictl_dip = dip;

        /* Initialize the cport/port mapping */
        for (i = 0; i < AHCI_MAX_PORTS; i++) {
                ahci_ctlp->ahcictl_port_to_cport[i] = 0xff;
                ahci_ctlp->ahcictl_cport_to_port[i] = 0xff;
        }

        attach_state |= AHCI_ATTACH_STATE_STATEP_ALLOC;

        /* Initialize FMA properties */
        ahci_fm_init(ahci_ctlp);

        attach_state |= AHCI_ATTACH_STATE_FMA;

        /*
         * Now map the AHCI base address; which includes global
         * registers and port control registers
         *
         * According to the spec, the AHCI Base Address is BAR5,
         * but BAR0-BAR4 are optional, so we need to check which
         * rnumber is used for BAR5.
         */

        /*
         * search through DDI "reg" property for the AHCI register set
         */
        if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, dip,
            DDI_PROP_DONTPASS, "reg", (int **)&regs,
            (uint_t *)&regs_length) != DDI_PROP_SUCCESS) {
                cmn_err(CE_WARN, "!ahci%d: Cannot lookup reg property",
                    instance);
                goto err_out;
        }

        /* AHCI Base Address is located at 0x24 offset */
        for (rnumber = 0; rnumber < regs_length; ++rnumber) {
                if ((regs[rnumber].pci_phys_hi & PCI_REG_REG_M)
                    == AHCI_PCI_RNUM)
                        break;
        }

        ddi_prop_free(regs);

        if (rnumber == regs_length) {
                cmn_err(CE_WARN, "!ahci%d: Cannot find AHCI register set",
                    instance);
                goto err_out;
        }

        AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "rnumber = %d", rnumber);

        status = ddi_regs_map_setup(dip,
            rnumber,
            (caddr_t *)&ahci_ctlp->ahcictl_ahci_addr,
            0,
            0,
            &accattr,
            &ahci_ctlp->ahcictl_ahci_acc_handle);
        if (status != DDI_SUCCESS) {
                cmn_err(CE_WARN, "!ahci%d: Cannot map register space",
                    instance);
                goto err_out;
        }

        attach_state |= AHCI_ATTACH_STATE_REG_MAP;

        /*
         * GHC.AE must be set to 1 before any other AHCI register
         * is accessed
         */
        ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
        ghc_control |= AHCI_HBA_GHC_AE;
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);

        /* Get the AHCI version information */
        ahci_version = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_VS(ahci_ctlp));

        cmn_err(CE_NOTE, "!ahci%d: hba AHCI version = %x.%x", instance,
            (ahci_version & 0xffff0000) >> 16,
            ((ahci_version & 0x0000ff00) >> 4 |
            (ahci_version & 0x000000ff)));

        /* We don't support controllers whose versions are lower than 1.0 */
        if (!(ahci_version & 0xffff0000)) {
                cmn_err(CE_WARN, "ahci%d: Don't support AHCI HBA with lower "
                    "than version 1.0", instance);
                goto err_out;
        }

        /* Get the HBA capabilities information */
        cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));

        AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba capabilities = 0x%x",
            cap_status);

        /* CAP2 (HBA Capabilities Extended) is available since AHCI spec 1.2 */
        if (ahci_version >= 0x00010200) {
                uint32_t cap2_status;

                /* Get the HBA capabilities extended information */
                cap2_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_GLOBAL_CAP2(ahci_ctlp));

                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "hba capabilities extended = 0x%x", cap2_status);

                if (cap2_status & AHCI_HBA_CAP2_SDS) {
                        ahci_ctlp->ahcictl_cap |= AHCI_CAP_SDS;
                }
        }

        if (cap_status & AHCI_HBA_CAP_EMS) {
                ahci_ctlp->ahcictl_cap |= AHCI_CAP_EMS;
                ahci_ctlp->ahcictl_em_loc =
                    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_GLOBAL_EM_LOC(ahci_ctlp));
                ahci_ctlp->ahcictl_em_ctl =
                    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
        }

#if AHCI_DEBUG
        /* Get the interface speed supported by the HBA */
        speed = (cap_status & AHCI_HBA_CAP_ISS) >> AHCI_HBA_CAP_ISS_SHIFT;
        if (speed == 0x01) {
                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "hba interface speed support: Gen 1 (1.5Gbps)", NULL);
        } else if (speed == 0x10) {
                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "hba interface speed support: Gen 2 (3 Gbps)", NULL);
        } else if (speed == 0x11) {
                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "hba interface speed support: Gen 3 (6 Gbps)", NULL);
        }
#endif

        /* Get the number of command slots supported by the HBA */
        ahci_ctlp->ahcictl_num_cmd_slots =
            ((cap_status & AHCI_HBA_CAP_NCS) >>
            AHCI_HBA_CAP_NCS_SHIFT) + 1;

        AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of cmd slots: %d",
            ahci_ctlp->ahcictl_num_cmd_slots);

        /* Get the bit map which indicates ports implemented by the HBA */
        ahci_ctlp->ahcictl_ports_implemented =
            ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_PI(ahci_ctlp));

        AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba implementation of ports: 0x%x",
            ahci_ctlp->ahcictl_ports_implemented);

        /* Max port number implemented */
        ahci_ctlp->ahcictl_num_ports =
            ddi_fls(ahci_ctlp->ahcictl_ports_implemented);

        AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "hba number of ports: %d",
            (cap_status & AHCI_HBA_CAP_NP) + 1);

        /* Get the number of implemented ports by the HBA */
        ahci_ctlp->ahcictl_num_implemented_ports =
            ahci_get_num_implemented_ports(
            ahci_ctlp->ahcictl_ports_implemented);

        AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
            "hba number of implemented ports: %d",
            ahci_ctlp->ahcictl_num_implemented_ports);

        /* Check whether HBA supports 64bit DMA addressing */
        if (!(cap_status & AHCI_HBA_CAP_S64A)) {
                ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
                ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "hba does not support 64-bit addressing", NULL);
        }

        /* Checking for the support of Port Multiplier */
        if (cap_status & AHCI_HBA_CAP_SPM) {
                ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_CBSS;
                AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
                    "hba supports port multiplier (CBSS)", NULL);

                /* Support FIS-based switching ? */
                if (cap_status & AHCI_HBA_CAP_FBSS) {
                        ahci_ctlp->ahcictl_cap |= AHCI_CAP_PMULT_FBSS;
                        AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
                            "hba supports FIS-based switching (FBSS)", NULL);
                }
        }

        /* Checking for Support Command List Override */
        if (cap_status & AHCI_HBA_CAP_SCLO) {
                ahci_ctlp->ahcictl_cap |= AHCI_CAP_SCLO;
                AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
                    "hba supports command list override.", NULL);
        }

        /* Checking for Asynchronous Notification */
        if (cap_status & AHCI_HBA_CAP_SSNTF) {
                ahci_ctlp->ahcictl_cap |= AHCI_CAP_SNTF;
                AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
                    "hba supports asynchronous notification.", NULL);
        }

        if (pci_config_setup(dip, &ahci_ctlp->ahcictl_pci_conf_handle)
            != DDI_SUCCESS) {
                cmn_err(CE_WARN, "!ahci%d: Cannot set up pci configure space",
                    instance);
                goto err_out;
        }

        attach_state |= AHCI_ATTACH_STATE_PCICFG_SETUP;

        /*
         * Check the pci configuration space, and set caps. We also
         * handle the hardware defect in this function.
         *
         * For example, force ATI SB600 to use 32-bit dma addressing
         * since it doesn't support 64-bit dma though its CAP register
         * declares it support.
         */
        if (ahci_config_space_init(ahci_ctlp) == AHCI_FAILURE) {
                cmn_err(CE_WARN, "!ahci%d: ahci_config_space_init failed",
                    instance);
                goto err_out;
        }

        /*
         * Disable the whole controller interrupts before adding
         * interrupt handlers(s).
         */
        ahci_disable_all_intrs(ahci_ctlp);

        /* Get supported interrupt types */
        if (ddi_intr_get_supported_types(dip, &intr_types) != DDI_SUCCESS) {
                cmn_err(CE_WARN, "!ahci%d: ddi_intr_get_supported_types failed",
                    instance);
                goto err_out;
        }

        AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
            "ddi_intr_get_supported_types() returned: 0x%x",
            intr_types);

        if (ahci_msi_enabled && (intr_types & DDI_INTR_TYPE_MSI)) {
                /*
                 * Try MSI first, but fall back to FIXED if failed
                 */
                if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_MSI) ==
                    DDI_SUCCESS) {
                        ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_MSI;
                        AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
                            "Using MSI interrupt type", NULL);
                        goto intr_done;
                }

                AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
                    "MSI registration failed, "
                    "trying FIXED interrupts", NULL);
        }

        if (intr_types & DDI_INTR_TYPE_FIXED) {
                if (ahci_add_intrs(ahci_ctlp, DDI_INTR_TYPE_FIXED) ==
                    DDI_SUCCESS) {
                        ahci_ctlp->ahcictl_intr_type = DDI_INTR_TYPE_FIXED;
                        AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
                            "Using FIXED interrupt type", NULL);
                        goto intr_done;
                }

                AHCIDBG(AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
                    "FIXED interrupt registration failed", NULL);
        }

        cmn_err(CE_WARN, "!ahci%d: Interrupt registration failed", instance);

        goto err_out;

intr_done:

        attach_state |= AHCI_ATTACH_STATE_INTR_ADDED;

        /* Initialize the controller mutex */
        mutex_init(&ahci_ctlp->ahcictl_mutex, NULL, MUTEX_DRIVER,
            (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);

        attach_state |= AHCI_ATTACH_STATE_MUTEX_INIT;

        if (ahci_dma_prdt_number < AHCI_MIN_PRDT_NUMBER) {
                ahci_dma_prdt_number = AHCI_MIN_PRDT_NUMBER;
        } else if (ahci_dma_prdt_number > AHCI_MAX_PRDT_NUMBER) {
                ahci_dma_prdt_number = AHCI_MAX_PRDT_NUMBER;
        }

        ahci_cmd_table_size = (sizeof (ahci_cmd_table_t) +
            (ahci_dma_prdt_number - AHCI_PRDT_NUMBER) *
            sizeof (ahci_prdt_item_t));

        AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
            "ahci_attach: ahci_dma_prdt_number set by user is 0x%x,"
            " ahci_cmd_table_size is 0x%x",
            ahci_dma_prdt_number, ahci_cmd_table_size);

        if (ahci_dma_prdt_number != AHCI_PRDT_NUMBER)
                ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_sgllen =
                    ahci_dma_prdt_number;

        ahci_ctlp->ahcictl_buffer_dma_attr = buffer_dma_attr;
        ahci_ctlp->ahcictl_rcvd_fis_dma_attr = rcvd_fis_dma_attr;
        ahci_ctlp->ahcictl_cmd_list_dma_attr = cmd_list_dma_attr;
        ahci_ctlp->ahcictl_cmd_table_dma_attr = cmd_table_dma_attr;

        /*
         * enable 64bit dma for data buffer for SB600 if
         * sb600_buf_64bit_dma_disable is B_FALSE
         */
        if ((ahci_buf_64bit_dma == B_FALSE) ||
            ((ahci_ctlp->ahcictl_cap & AHCI_CAP_BUF_32BIT_DMA) &&
            !(sb600_buf_64bit_dma_disable == B_FALSE &&
            ahci_ctlp->ahcictl_venid == 0x1002 &&
            ahci_ctlp->ahcictl_devid == 0x4380))) {
                ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_addr_hi =
                    0xffffffffull;
        }

        /*
         * enable 64bit dma for command buffer for SB600/700/710/800
         * if sbxxx_commu_64bit_dma_disable is B_FALSE
         */
        if ((ahci_commu_64bit_dma == B_FALSE) ||
            ((ahci_ctlp->ahcictl_cap & AHCI_CAP_COMMU_32BIT_DMA) &&
            !(sbxxx_commu_64bit_dma_disable == B_FALSE &&
            ahci_ctlp->ahcictl_venid == 0x1002 &&
            (ahci_ctlp->ahcictl_devid == 0x4380 ||
            ahci_ctlp->ahcictl_devid == 0x4391)))) {
                ahci_ctlp->ahcictl_rcvd_fis_dma_attr.dma_attr_addr_hi =
                    0xffffffffull;
                ahci_ctlp->ahcictl_cmd_list_dma_attr.dma_attr_addr_hi =
                    0xffffffffull;
                ahci_ctlp->ahcictl_cmd_table_dma_attr.dma_attr_addr_hi =
                    0xffffffffull;
        }

        /* Allocate the ports structure */
        status = ahci_alloc_ports_state(ahci_ctlp);
        if (status != AHCI_SUCCESS) {
                cmn_err(CE_WARN, "!ahci%d: Cannot allocate ports structure",
                    instance);
                goto err_out;
        }

        attach_state |= AHCI_ATTACH_STATE_PORT_ALLOC;

        /*
         * Initialize the controller and ports.
         */
        status = ahci_initialize_controller(ahci_ctlp);
        if (status != AHCI_SUCCESS) {
                cmn_err(CE_WARN, "!ahci%d: HBA initialization failed",
                    instance);
                goto err_out;
        }

        attach_state |= AHCI_ATTACH_STATE_HW_INIT;

        /* Start one thread to check packet timeouts */
        ahci_ctlp->ahcictl_timeout_id = timeout(
            (void (*)(void *))ahci_watchdog_handler,
            (caddr_t)ahci_ctlp, ahci_watchdog_tick);

        attach_state |= AHCI_ATTACH_STATE_TIMEOUT_ENABLED;

        if (!ahci_em_init(ahci_ctlp)) {
                cmn_err(CE_WARN, "!ahci%d: failed to initialize enclosure "
                    "services", instance);
                goto err_out;
        }
        attach_state |= AHCI_ATTACH_STATE_ENCLOSURE;

        if (ahci_register_sata_hba_tran(ahci_ctlp, cap_status)) {
                cmn_err(CE_WARN, "!ahci%d: sata hba tran registration failed",
                    instance);
                goto err_out;
        }

        /* Check all handles at the end of the attach operation. */
        if (ahci_check_all_handle(ahci_ctlp) != DDI_SUCCESS) {
                cmn_err(CE_WARN, "!ahci%d: invalid dma/acc handles",
                    instance);
                goto err_out;
        }

        ahci_ctlp->ahcictl_flags &= ~AHCI_ATTACH;

        AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "ahci_attach success!", NULL);

        return (DDI_SUCCESS);

err_out:
        /* FMA message */
        ahci_fm_ereport(ahci_ctlp, DDI_FM_DEVICE_NO_RESPONSE);
        ddi_fm_service_impact(ahci_ctlp->ahcictl_dip, DDI_SERVICE_LOST);

        if (attach_state & AHCI_ATTACH_STATE_ENCLOSURE) {
                ahci_em_fini(ahci_ctlp);
        }

        if (attach_state & AHCI_ATTACH_STATE_TIMEOUT_ENABLED) {
                mutex_enter(&ahci_ctlp->ahcictl_mutex);
                (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
                ahci_ctlp->ahcictl_timeout_id = 0;
                mutex_exit(&ahci_ctlp->ahcictl_mutex);
        }

        if (attach_state & AHCI_ATTACH_STATE_HW_INIT) {
                ahci_uninitialize_controller(ahci_ctlp);
        }

        if (attach_state & AHCI_ATTACH_STATE_PORT_ALLOC) {
                ahci_dealloc_ports_state(ahci_ctlp);
        }

        if (attach_state & AHCI_ATTACH_STATE_MUTEX_INIT) {
                mutex_destroy(&ahci_ctlp->ahcictl_mutex);
        }

        if (attach_state & AHCI_ATTACH_STATE_INTR_ADDED) {
                ahci_rem_intrs(ahci_ctlp);
        }

        if (attach_state & AHCI_ATTACH_STATE_PCICFG_SETUP) {
                pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);
        }

        if (attach_state & AHCI_ATTACH_STATE_REG_MAP) {
                ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);
        }

        if (attach_state & AHCI_ATTACH_STATE_FMA) {
                ahci_fm_fini(ahci_ctlp);
        }

        if (attach_state & AHCI_ATTACH_STATE_STATEP_ALLOC) {
                ddi_soft_state_free(ahci_statep, instance);
        }

        return (DDI_FAILURE);
}

/*
 * The detach entry point for dev_ops.
 */
static int
ahci_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
        ahci_ctl_t *ahci_ctlp;
        int instance;
        int ret;

        instance = ddi_get_instance(dip);
        ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_detach enter", NULL);

        switch (cmd) {
        case DDI_DETACH:

                /* disable the interrupts for an uninterrupted detach */
                mutex_enter(&ahci_ctlp->ahcictl_mutex);
                ahci_disable_all_intrs(ahci_ctlp);
                mutex_exit(&ahci_ctlp->ahcictl_mutex);

                /* unregister from the sata framework. */
                ret = ahci_unregister_sata_hba_tran(ahci_ctlp);
                if (ret != AHCI_SUCCESS) {
                        mutex_enter(&ahci_ctlp->ahcictl_mutex);
                        ahci_enable_all_intrs(ahci_ctlp);
                        mutex_exit(&ahci_ctlp->ahcictl_mutex);
                        return (DDI_FAILURE);
                }

                ahci_em_fini(ahci_ctlp);

                mutex_enter(&ahci_ctlp->ahcictl_mutex);

                /* stop the watchdog handler */
                (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
                ahci_ctlp->ahcictl_timeout_id = 0;

                mutex_exit(&ahci_ctlp->ahcictl_mutex);

                /* uninitialize the controller */
                ahci_uninitialize_controller(ahci_ctlp);

                /* remove the interrupts */
                ahci_rem_intrs(ahci_ctlp);

                /* deallocate the ports structures */
                ahci_dealloc_ports_state(ahci_ctlp);

                /* destroy mutex */
                mutex_destroy(&ahci_ctlp->ahcictl_mutex);

                /* teardown the pci config */
                pci_config_teardown(&ahci_ctlp->ahcictl_pci_conf_handle);

                /* remove the reg maps. */
                ddi_regs_map_free(&ahci_ctlp->ahcictl_ahci_acc_handle);

                /* release fma resource */
                ahci_fm_fini(ahci_ctlp);

                /* free the soft state. */
                ddi_soft_state_free(ahci_statep, instance);

                return (DDI_SUCCESS);

        case DDI_SUSPEND:

                /*
                 * The steps associated with suspension must include putting
                 * the underlying device into a quiescent state so that it
                 * will not generate interrupts or modify or access memory.
                 */
                mutex_enter(&ahci_ctlp->ahcictl_mutex);
                if (ahci_ctlp->ahcictl_flags & AHCI_SUSPEND) {
                        mutex_exit(&ahci_ctlp->ahcictl_mutex);
                        return (DDI_SUCCESS);
                }

                ahci_ctlp->ahcictl_flags |= AHCI_SUSPEND;

                /* stop the watchdog handler */
                if (ahci_ctlp->ahcictl_timeout_id) {
                        (void) untimeout(ahci_ctlp->ahcictl_timeout_id);
                        ahci_ctlp->ahcictl_timeout_id = 0;
                }

                mutex_exit(&ahci_ctlp->ahcictl_mutex);

                ahci_em_suspend(ahci_ctlp);

                /*
                 * drain the taskq
                 */
                ahci_drain_ports_taskq(ahci_ctlp);

                /*
                 * Disable the interrupts and stop all the ports.
                 */
                ahci_uninitialize_controller(ahci_ctlp);

                return (DDI_SUCCESS);

        default:
                return (DDI_FAILURE);
        }
}

/*
 * The info entry point for dev_ops.
 *
 */
static int
ahci_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd,
    void *arg, void **result)
{
#ifndef __lock_lint
        _NOTE(ARGUNUSED(dip))
#endif /* __lock_lint */

        ahci_ctl_t *ahci_ctlp;
        int instance;
        dev_t dev;

        dev = (dev_t)arg;
        instance = getminor(dev);

        switch (infocmd) {
                case DDI_INFO_DEVT2DEVINFO:
                        ahci_ctlp = ddi_get_soft_state(ahci_statep,  instance);
                        if (ahci_ctlp != NULL) {
                                *result = ahci_ctlp->ahcictl_dip;
                                return (DDI_SUCCESS);
                        } else {
                                *result = NULL;
                                return (DDI_FAILURE);
                        }
                case DDI_INFO_DEVT2INSTANCE:
                        *(int *)result = instance;
                        break;
                default:
                        break;
        }

        return (DDI_SUCCESS);
}

/*
 * Registers the ahci with sata framework.
 */
static int
ahci_register_sata_hba_tran(ahci_ctl_t *ahci_ctlp, uint32_t cap_status)
{
        struct  sata_hba_tran   *sata_hba_tran;

        AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_register_sata_hba_tran enter", NULL);

        mutex_enter(&ahci_ctlp->ahcictl_mutex);

        /* Allocate memory for the sata_hba_tran  */
        sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP);

        sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV;
        sata_hba_tran->sata_tran_hba_dip = ahci_ctlp->ahcictl_dip;
        sata_hba_tran->sata_tran_hba_dma_attr =
            &ahci_ctlp->ahcictl_buffer_dma_attr;

        /* Report the number of implemented ports */
        sata_hba_tran->sata_tran_hba_num_cports =
            ahci_ctlp->ahcictl_num_implemented_ports;

        /* Support ATAPI device */
        sata_hba_tran->sata_tran_hba_features_support = SATA_CTLF_ATAPI;

        /* Get the data transfer capability for PIO command by the HBA */
        if (cap_status & AHCI_HBA_CAP_PMD) {
                ahci_ctlp->ahcictl_cap |= AHCI_CAP_PIO_MDRQ;
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports multiple "
                    "DRQ block data transfer for PIO command protocol", NULL);
        }

        /*
         * According to the AHCI spec, the ATA/ATAPI-7 queued feature set
         * is not supported by AHCI (including the READ QUEUED (EXT), WRITE
         * QUEUED (EXT), and SERVICE commands). Queued operations are
         * supported in AHCI using the READ FPDMA QUEUED and WRITE FPDMA
         * QUEUED commands when the HBA and device support native command
         * queuing(NCQ).
         *
         * SATA_CTLF_NCQ will be set to sata_tran_hba_features_support if the
         * CAP register of the HBA indicates NCQ is supported.
         *
         * SATA_CTLF_NCQ cannot be set if AHCI_CAP_NO_MCMDLIST_NONQUEUE is
         * set because the previous register content of PxCI can be re-written
         * in the register write.
         */
        if ((cap_status & AHCI_HBA_CAP_SNCQ) &&
            !(ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE)) {
                sata_hba_tran->sata_tran_hba_features_support |= SATA_CTLF_NCQ;
                ahci_ctlp->ahcictl_cap |= AHCI_CAP_NCQ;
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "HBA supports Native "
                    "Command Queuing", NULL);
        }

        /* Support port multiplier? */
        if (cap_status & AHCI_HBA_CAP_SPM) {
                sata_hba_tran->sata_tran_hba_features_support |=
                    SATA_CTLF_PORT_MULTIPLIER;

                /* Support FIS-based switching for port multiplier? */
                if (cap_status & AHCI_HBA_CAP_FBSS) {
                        sata_hba_tran->sata_tran_hba_features_support |=
                            SATA_CTLF_PMULT_FBS;
                }
        }

        /* Report the number of command slots */
        sata_hba_tran->sata_tran_hba_qdepth = ahci_ctlp->ahcictl_num_cmd_slots;

        sata_hba_tran->sata_tran_probe_port = ahci_tran_probe_port;
        sata_hba_tran->sata_tran_start = ahci_tran_start;
        sata_hba_tran->sata_tran_abort = ahci_tran_abort;
        sata_hba_tran->sata_tran_reset_dport = ahci_tran_reset_dport;
        sata_hba_tran->sata_tran_hotplug_ops = &ahci_tran_hotplug_ops;
#ifdef __lock_lint
        sata_hba_tran->sata_tran_selftest = ahci_selftest;
#endif
        /*
         * When SATA framework adds support for pwrmgt the
         * pwrmgt_ops needs to be updated
         */
        sata_hba_tran->sata_tran_pwrmgt_ops = NULL;
        sata_hba_tran->sata_tran_ioctl = ahci_em_ioctl;

        ahci_ctlp->ahcictl_sata_hba_tran = sata_hba_tran;

        mutex_exit(&ahci_ctlp->ahcictl_mutex);

        /* Attach it to SATA framework */
        if (sata_hba_attach(ahci_ctlp->ahcictl_dip, sata_hba_tran, DDI_ATTACH)
            != DDI_SUCCESS) {
                kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t));
                mutex_enter(&ahci_ctlp->ahcictl_mutex);
                ahci_ctlp->ahcictl_sata_hba_tran = NULL;
                mutex_exit(&ahci_ctlp->ahcictl_mutex);
                return (AHCI_FAILURE);
        }

        return (AHCI_SUCCESS);
}

/*
 * Unregisters the ahci with sata framework.
 */
static int
ahci_unregister_sata_hba_tran(ahci_ctl_t *ahci_ctlp)
{
        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_unregister_sata_hba_tran enter", NULL);

        /* Detach from the SATA framework. */
        if (sata_hba_detach(ahci_ctlp->ahcictl_dip, DDI_DETACH) !=
            DDI_SUCCESS) {
                return (AHCI_FAILURE);
        }

        /* Deallocate sata_hba_tran. */
        kmem_free((void *)ahci_ctlp->ahcictl_sata_hba_tran,
            sizeof (sata_hba_tran_t));

        mutex_enter(&ahci_ctlp->ahcictl_mutex);
        ahci_ctlp->ahcictl_sata_hba_tran = NULL;
        mutex_exit(&ahci_ctlp->ahcictl_mutex);

        return (AHCI_SUCCESS);
}

#define SET_PORTSTR(str, addrp)                                         \
        if (AHCI_ADDR_IS_PORT(addrp))                                   \
                (void) sprintf((str), "%d", (addrp)->aa_port);          \
        else if (AHCI_ADDR_IS_PMULT(addrp))                             \
                (void) sprintf((str), "%d (pmult)", (addrp)->aa_port);  \
        else                                                            \
                (void) sprintf((str), "%d:%d", (addrp)->aa_port,        \
                    (addrp)->aa_pmport);

/*
 * ahci_tran_probe_port is called by SATA framework. It returns port state,
 * port status registers and an attached device type via sata_device
 * structure.
 *
 * We return the cached information from a previous hardware probe. The
 * actual hardware probing itself was done either from within
 * ahci_initialize_controller() during the driver attach or from a phy
 * ready change interrupt handler.
 */
static int
ahci_tran_probe_port(dev_info_t *dip, sata_device_t *sd)
{
        ahci_ctl_t *ahci_ctlp;
        ahci_port_t *ahci_portp;
        ahci_addr_t addr, pmult_addr;
        uint8_t cport = sd->satadev_addr.cport;
        char portstr[10];
        uint8_t device_type;
        uint32_t port_state;
        uint8_t port;
        int rval = SATA_SUCCESS, rval_init;

        port_state = 0;
        ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
        port = ahci_ctlp->ahcictl_cport_to_port[cport];

        ahci_portp = ahci_ctlp->ahcictl_ports[port];

        mutex_enter(&ahci_portp->ahciport_mutex);

        ahci_get_ahci_addr(ahci_ctlp, sd, &addr);
        ASSERT(AHCI_ADDR_IS_VALID(&addr));
        SET_PORTSTR(portstr, &addr);

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_tran_probe_port enter: port %s", portstr);

        if ((AHCI_ADDR_IS_PMULT(&addr) || AHCI_ADDR_IS_PMPORT(&addr)) &&
            (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
            ahci_portp->ahciport_pmult_info == NULL)) {
                /* port mutliplier is removed. */
                AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
                    "ahci_tran_probe_port: "
                    "pmult is removed from port %s", portstr);
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (SATA_FAILURE);
        }

        /*
         * The sata_device may refer to
         * 1. A controller port.
         *    A controller port should be ready here.
         * 2. A port multiplier.
         *    SATA_ADDR_PMULT_SPEC - if it is not initialized yet, initialize
         *    it and register the port multiplier to the framework.
         *    SATA_ADDR_PMULT - check the status of all its device ports.
         * 3. A port multiplier port.
         *    If it has not been initialized, initialized it.
         *
         * A port multiplier or a port multiplier port may require some
         * initialization because we cannot do these time-consuming jobs in an
         * interrupt context.
         */
        if (sd->satadev_addr.qual & SATA_ADDR_PMULT_SPEC) {
                AHCI_ADDR_SET_PMULT(&pmult_addr, port);
                /* Initialize registers on a port multiplier */
                rval_init = ahci_initialize_pmult(ahci_ctlp,
                    ahci_portp, &pmult_addr, sd);
                if (rval_init != AHCI_SUCCESS) {
                        AHCIDBG(AHCIDBG_PMULT, ahci_ctlp,
                            "ahci_tran_probe_port: "
                            "pmult initialization failed.", NULL);
                        mutex_exit(&ahci_portp->ahciport_mutex);
                        return (SATA_FAILURE);
                }
        } else if (sd->satadev_addr.qual & SATA_ADDR_PMULT) {
                /* Check pmports hotplug events */
                (void) ahci_probe_pmult(ahci_ctlp, ahci_portp, &addr);
        } else if (sd->satadev_addr.qual & (SATA_ADDR_PMPORT |
            SATA_ADDR_DPMPORT)) {
                if (ahci_probe_pmport(ahci_ctlp, ahci_portp,
                    &addr, sd) != AHCI_SUCCESS) {
                        rval = SATA_FAILURE;
                        goto out;
                }
        }

        /* Update port state and device type */
        port_state = AHCIPORT_GET_STATE(ahci_portp, &addr);

        switch (port_state) {

        case SATA_PSTATE_FAILED:
                sd->satadev_state = SATA_PSTATE_FAILED;
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_tran_probe_port: port %s PORT FAILED", portstr);
                goto out;

        case SATA_PSTATE_SHUTDOWN:
                sd->satadev_state = SATA_PSTATE_SHUTDOWN;
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_tran_probe_port: port %s PORT SHUTDOWN", portstr);
                goto out;

        case SATA_PSTATE_PWROFF:
                sd->satadev_state = SATA_PSTATE_PWROFF;
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_tran_probe_port: port %s PORT PWROFF", portstr);
                goto out;

        case SATA_PSTATE_PWRON:
                sd->satadev_state = SATA_PSTATE_PWRON;
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                    "ahci_tran_probe_port: port %s PORT PWRON", portstr);
                break;

        default:
                sd->satadev_state = port_state;
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                    "ahci_tran_probe_port: port %s PORT NORMAL %x",
                    portstr, port_state);
                break;
        }

        device_type = AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);

        switch (device_type) {

        case SATA_DTYPE_ATADISK:
                sd->satadev_type = SATA_DTYPE_ATADISK;
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                    "ahci_tran_probe_port: port %s DISK found", portstr);
                break;

        case SATA_DTYPE_ATAPI:
                /*
                 * HBA driver only knows it's an ATAPI device, and don't know
                 * it's CD/DVD, tape or ATAPI disk because the ATAPI device
                 * type need to be determined by checking IDENTIFY PACKET
                 * DEVICE data
                 */
                sd->satadev_type = SATA_DTYPE_ATAPI;
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                    "ahci_tran_probe_port: port %s ATAPI found", portstr);
                break;

        case SATA_DTYPE_PMULT:
                ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr));
                sd->satadev_type = SATA_DTYPE_PMULT;

                /* Update the number of pmports. */
                ASSERT(ahci_portp->ahciport_pmult_info != NULL);
                sd->satadev_add_info = ahci_portp->
                    ahciport_pmult_info->ahcipmi_num_dev_ports;

                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                    "ahci_tran_probe_port: port %s Port Multiplier found",
                    portstr);
                break;

        case SATA_DTYPE_UNKNOWN:
                sd->satadev_type = SATA_DTYPE_UNKNOWN;
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                    "ahci_tran_probe_port: port %s Unknown device found",
                    portstr);
                break;

        default:
                /* we don't support any other device types */
                sd->satadev_type = SATA_DTYPE_NONE;
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                    "ahci_tran_probe_port: port %s No device found", portstr);
                break;
        }

out:
        /* Register update only fails while probing a pmult/pmport */
        if (AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMULT(&addr)) {
                ahci_update_sata_registers(ahci_ctlp, port, sd);
        } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
                if (port_state & SATA_STATE_READY)
                        if (ahci_update_pmult_pscr(ahci_ctlp,
                            &addr, sd) != AHCI_SUCCESS)
                                rval = SATA_FAILURE;
        }

        /* Check handles for the sata registers access */
        if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
            (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
                ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
                    DDI_SERVICE_UNAFFECTED);
                rval = SATA_FAILURE;
        }

        mutex_exit(&ahci_portp->ahciport_mutex);
        return (rval);
}

/*
 * There are four operation modes in sata framework:
 * SATA_OPMODE_INTERRUPTS
 * SATA_OPMODE_POLLING
 * SATA_OPMODE_ASYNCH
 * SATA_OPMODE_SYNCH
 *
 * Their combined meanings as following:
 *
 * SATA_OPMODE_SYNCH
 * The command has to be completed before sata_tran_start functions returns.
 * Either interrupts or polling could be used - it's up to the driver.
 * Mode used currently for internal, sata-module initiated operations.
 *
 * SATA_OPMODE_SYNCH | SATA_OPMODE_INTERRUPTS
 * It is the same as the one above.
 *
 * SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING
 * The command has to be completed before sata_tran_start function returns.
 * No interrupt used, polling only. This should be the mode used for scsi
 * packets with FLAG_NOINTR.
 *
 * SATA_OPMODE_ASYNCH | SATA_OPMODE_INTERRUPTS
 * The command may be queued (callback function specified). Interrupts could
 * be used. It's normal operation mode.
 */
/*
 * Called by sata framework to transport a sata packet down stream.
 */
static int
ahci_tran_start(dev_info_t *dip, sata_pkt_t *spkt)
{
        ahci_ctl_t *ahci_ctlp;
        ahci_port_t *ahci_portp;
        ahci_addr_t addr;
        uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
        uint8_t port;
        char portstr[10];

        ahci_ctlp = ddi_get_soft_state(ahci_statep, ddi_get_instance(dip));
        port = ahci_ctlp->ahcictl_cport_to_port[cport];

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_tran_start enter: cport %d satapkt 0x%p",
            cport, (void *)spkt);

        ahci_portp = ahci_ctlp->ahcictl_ports[port];

        mutex_enter(&ahci_portp->ahciport_mutex);
        ahci_get_ahci_addr(ahci_ctlp, &spkt->satapkt_device, &addr);
        SET_PORTSTR(portstr, &addr);

        /* Sanity check */
        if (AHCI_ADDR_IS_PMPORT(&addr)) {
                if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT ||
                    ahci_portp->ahciport_pmult_info == NULL) {

                        spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
                        spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
                        spkt->satapkt_device.satadev_state = SATA_STATE_UNKNOWN;
                        ahci_update_sata_registers(ahci_ctlp, port,
                            &spkt->satapkt_device);
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_tran_start returning PORT_ERROR while "
                            "pmult removed: port: %s", portstr);
                        mutex_exit(&ahci_portp->ahciport_mutex);
                        return (SATA_TRAN_PORT_ERROR);
                }

                if (!(AHCIPORT_GET_STATE(ahci_portp, &addr) &
                    SATA_STATE_READY)) {
                        if (!ddi_in_panic() ||
                            ahci_initialize_pmport(ahci_ctlp,
                            ahci_portp, &addr) != AHCI_SUCCESS) {
                                spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
                                spkt->satapkt_device.satadev_type =
                                    AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
                                spkt->satapkt_device.satadev_state =
                                    AHCIPORT_GET_STATE(ahci_portp, &addr);
                                ahci_update_sata_registers(ahci_ctlp, port,
                                    &spkt->satapkt_device);
                                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                                    "ahci_tran_start returning PORT_ERROR "
                                    "while sub-link is not initialized "
                                    "at port: %s", portstr);
                                mutex_exit(&ahci_portp->ahciport_mutex);
                                return (SATA_TRAN_PORT_ERROR);
                        }
                }
        }

        if (AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_FAILED ||
            AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_SHUTDOWN||
            AHCIPORT_GET_STATE(ahci_portp, &addr) & SATA_PSTATE_PWROFF) {
                /*
                 * In case the target driver would send the packet before
                 * sata framework can have the opportunity to process those
                 * event reports.
                 */
                spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
                spkt->satapkt_device.satadev_state =
                    ahci_portp->ahciport_port_state;
                ahci_update_sata_registers(ahci_ctlp, port,
                    &spkt->satapkt_device);
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_tran_start returning PORT_ERROR while "
                    "port in FAILED/SHUTDOWN/PWROFF state: "
                    "port: %s", portstr);
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (SATA_TRAN_PORT_ERROR);
        }

        if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) == SATA_DTYPE_NONE) {
                /*
                 * ahci_intr_phyrdy_change() may have rendered it to
                 * SATA_DTYPE_NONE.
                 */
                spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
                spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
                spkt->satapkt_device.satadev_state =
                    ahci_portp->ahciport_port_state;
                ahci_update_sata_registers(ahci_ctlp, port,
                    &spkt->satapkt_device);
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_tran_start returning PORT_ERROR while "
                    "no device attached: port: %s", portstr);
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (SATA_TRAN_PORT_ERROR);
        }

        /* R/W PMULT command will occupy the whole HBA port */
        if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_tran_start returning BUSY while "
                    "executing READ/WRITE PORT-MULT command: "
                    "port: %s", portstr);
                spkt->satapkt_reason = SATA_PKT_BUSY;
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (SATA_TRAN_BUSY);
        }

        if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_tran_start returning BUSY while "
                    "hot-plug in progress: port: %s", portstr);
                spkt->satapkt_reason = SATA_PKT_BUSY;
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (SATA_TRAN_BUSY);
        }

        /*
         * SATA HBA driver should remember that a device was reset and it
         * is supposed to reject any packets which do not specify either
         * SATA_IGNORE_DEV_RESET_STATE or SATA_CLEAR_DEV_RESET_STATE.
         *
         * This is to prevent a race condition when a device was arbitrarily
         * reset by the HBA driver (and lost it's setting) and a target
         * driver sending some commands to a device before the sata framework
         * has a chance to restore the device setting (such as cache enable/
         * disable or other resettable stuff).
         */
        /*
         * It is unnecessary to use specific flags to indicate
         * reset_in_progress for a pmport. While mopping, all command will be
         * mopped so that the entire HBA port is being dealt as a single
         * object.
         */
        if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) {
                ahci_portp->ahciport_reset_in_progress = 0;
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_tran_start [CLEAR] the "
                    "reset_in_progress for port: %d", port);
        }

        if (ahci_portp->ahciport_reset_in_progress &&
            ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset &&
            ! ddi_in_panic()) {
                spkt->satapkt_reason = SATA_PKT_BUSY;
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_tran_start returning BUSY while "
                    "reset in progress: port: %d", port);
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (SATA_TRAN_BUSY);
        }

#ifdef AHCI_DEBUG
        if (spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_tran_start: packet 0x%p [PASSTHRU] at port %d",
                    spkt, port);
        }
#endif

        if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
                spkt->satapkt_reason = SATA_PKT_BUSY;
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_tran_start returning BUSY while "
                    "mopping in progress: port: %d", port);
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (SATA_TRAN_BUSY);
        }

        if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
                ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
                    DDI_SERVICE_UNAFFECTED);
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (SATA_TRAN_BUSY);
        }

        if (spkt->satapkt_op_mode &
            (SATA_OPMODE_SYNCH | SATA_OPMODE_POLLING)) {
                /*
                 * If a SYNC command to be executed in interrupt context,
                 * bounce it back to sata module.
                 */
                if (!(spkt->satapkt_op_mode & SATA_OPMODE_POLLING) &&
                    servicing_interrupt()) {
                        spkt->satapkt_reason = SATA_PKT_BUSY;
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_tran_start returning BUSY while "
                            "sending SYNC mode under interrupt context: "
                            "port : %d", port);
                        mutex_exit(&ahci_portp->ahciport_mutex);
                        return (SATA_TRAN_BUSY);
                }

                /* We need to do the sync start now */
                if (ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr,
                    spkt) == AHCI_FAILURE) {
                        goto fail_out;
                }
        } else {
                /* Async start, using interrupt */
                if (ahci_deliver_satapkt(ahci_ctlp, ahci_portp, &addr, spkt)
                    == AHCI_FAILURE) {
                        spkt->satapkt_reason = SATA_PKT_QUEUE_FULL;
                        goto fail_out;
                }
        }

        AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_tran_start "
            "sata tran accepted: port %s", portstr);

        mutex_exit(&ahci_portp->ahciport_mutex);
        return (SATA_TRAN_ACCEPTED);

fail_out:
        /*
         * Failed to deliver packet to the controller.
         * Check if it's caused by invalid handles.
         */
        if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS ||
            ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
                spkt->satapkt_device.satadev_type =
                    AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr);
                spkt->satapkt_device.satadev_state =
                    AHCIPORT_GET_STATE(ahci_portp, &addr);
                spkt->satapkt_reason = SATA_PKT_DEV_ERROR;
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (SATA_TRAN_PORT_ERROR);
        }

        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_tran_start "
            "return QUEUE_FULL: port %d", port);
        mutex_exit(&ahci_portp->ahciport_mutex);
        return (SATA_TRAN_QUEUE_FULL);
}

/*
 * SATA_OPMODE_SYNCH flag is set
 *
 * If SATA_OPMODE_POLLING flag is set, then we must poll the command
 * without interrupt, otherwise we can still use the interrupt.
 */
static int
ahci_do_sync_start(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    ahci_addr_t *addrp, sata_pkt_t *spkt)
{
        int pkt_timeout_ticks;
        uint32_t timeout_tags;
        int rval;
        int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
        uint8_t port = addrp->aa_port;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_do_sync_start enter: "
            "port %d:%d spkt 0x%p", port, addrp->aa_pmport, spkt);

        if (spkt->satapkt_op_mode & SATA_OPMODE_POLLING) {
                ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_POLLING;
                if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
                    addrp, spkt)) == AHCI_FAILURE) {
                        ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
                        return (rval);
                }

                pkt_timeout_ticks =
                    drv_usectohz((clock_t)spkt->satapkt_time * 1000000);

                while (spkt->satapkt_reason == SATA_PKT_BUSY) {
                        /* Simulate the interrupt */
                        mutex_exit(&ahci_portp->ahciport_mutex);
                        ahci_port_intr(ahci_ctlp, ahci_portp, port);
                        mutex_enter(&ahci_portp->ahciport_mutex);

                        if (spkt->satapkt_reason != SATA_PKT_BUSY)
                                break;

                        mutex_exit(&ahci_portp->ahciport_mutex);
                        drv_usecwait(AHCI_1MS_USECS);
                        mutex_enter(&ahci_portp->ahciport_mutex);

                        pkt_timeout_ticks -= AHCI_1MS_TICKS;
                        if (pkt_timeout_ticks < 0) {
                                cmn_err(CE_WARN, "!ahci%d: ahci_do_sync_start "
                                    "port %d satapkt 0x%p timed out\n",
                                    instance, port, (void *)spkt);
                                timeout_tags = (0x1 << rval);
                                mutex_exit(&ahci_portp->ahciport_mutex);
                                ahci_timeout_pkts(ahci_ctlp, ahci_portp,
                                    port, timeout_tags);
                                mutex_enter(&ahci_portp->ahciport_mutex);
                        }
                }

                ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_POLLING;
                return (AHCI_SUCCESS);

        } else {
                if ((rval = ahci_deliver_satapkt(ahci_ctlp, ahci_portp,
                    addrp, spkt)) == AHCI_FAILURE)
                        return (rval);

#if AHCI_DEBUG
                /*
                 * Note that the driver always uses the slot 0 to deliver
                 * REQUEST SENSE or READ LOG EXT command
                 */
                if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
                        ASSERT(rval == 0);
#endif

                while (spkt->satapkt_reason == SATA_PKT_BUSY)
                        cv_wait(&ahci_portp->ahciport_cv,
                            &ahci_portp->ahciport_mutex);

                return (AHCI_SUCCESS);
        }
}

/*
 * Searches for and claims a free command slot.
 *
 * Returns value:
 *
 * AHCI_FAILURE returned only if
 *      1. No empty slot left
 *      2. Non-queued command requested while queued command(s) is outstanding
 *      3. Queued command requested while non-queued command(s) is outstanding
 *      4. HBA doesn't support multiple-use of command list while already a
 *         non-queued command is oustanding
 *      5. Queued command requested while some queued command(s) has been
 *         outstanding on a different port multiplier port. (AHCI spec 1.2,
 *         9.1.2)
 *
 * claimed slot number returned if succeeded
 *
 * NOTE: it will always return slot 0 for following commands to simplify the
 * algorithm.
 *      1. REQUEST SENSE or READ LOG EXT command during error recovery process
 *      2. READ/WRITE PORTMULT command
 */
static int
ahci_claim_free_slot(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    ahci_addr_t *addrp, int command_type)
{
        uint32_t port_cmd_issue;
        uint32_t free_slots;
        int slot;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_claim_free_slot enter "
            "ahciport_pending_tags = 0x%x "
            "ahciport_pending_ncq_tags = 0x%x",
            ahci_portp->ahciport_pending_tags,
            ahci_portp->ahciport_pending_ncq_tags);

        free_slots = 0;
        /*
         * According to the AHCI spec, system software is responsible to
         * ensure that queued and non-queued commands are not mixed in
         * the command list.
         */
        if (command_type == AHCI_NON_NCQ_CMD) {
                /* Non-NCQ command request */
                if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                        AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
                            "ahci_claim_free_slot: there is still pending "
                            "queued command(s) in the command list, "
                            "so no available slot for the non-queued "
                            "command", NULL);
                        return (AHCI_FAILURE);
                }
                if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
                        AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
                            "ahci_claim_free_slot: there is still pending "
                            "read/write port-mult command(s) in command list, "
                            "so no available slot for the non-queued command",
                            NULL);
                        return (AHCI_FAILURE);
                }
                if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_NO_MCMDLIST_NONQUEUE) &&
                    NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                        AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                            "ahci_claim_free_slot: HBA cannot support multiple-"
                            "use of the command list for non-queued commands",
                            NULL);
                        return (AHCI_FAILURE);
                }
                free_slots = (~ahci_portp->ahciport_pending_tags) &
                    AHCI_SLOT_MASK(ahci_ctlp);
        } else if (command_type == AHCI_NCQ_CMD) {
                /* NCQ command request */
                if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                        AHCIDBG(AHCIDBG_INFO|AHCIDBG_NCQ, ahci_ctlp,
                            "ahci_claim_free_slot: there is still pending "
                            "non-queued command(s) in the command list, "
                            "so no available slot for the queued command",
                            NULL);
                        return (AHCI_FAILURE);
                }

                /*
                 * NCQ commands cannot be sent to different port multiplier
                 * ports in Command-Based Switching mode
                 */
                /*
                 * NOTE: In Command-Based Switching mode, AHCI controller
                 * usually reports a 'Handshake Error' when multiple NCQ
                 * commands are outstanding simultaneously.
                 */
                if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
                        ASSERT(ahci_portp->ahciport_pmult_info != NULL);
                        if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS) &&
                            NCQ_CMD_IN_PROGRESS(ahci_portp) &&
                            AHCIPORT_NCQ_PMPORT(ahci_portp) !=
                            addrp->aa_pmport) {
                                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                                    "ahci_claim_free_slot: there is still "
                                    "pending queued command(s) in the "
                                    "command list for another Port Multiplier "
                                    "port, so no available slot.", NULL);
                                return (AHCI_FAILURE);
                        }
                }

                free_slots = (~ahci_portp->ahciport_pending_ncq_tags) &
                    AHCI_NCQ_SLOT_MASK(ahci_portp);
        } else if (command_type == AHCI_ERR_RETRI_CMD) {
                /* Error retrieval command request */
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_claim_free_slot: slot 0 is allocated for REQUEST "
                    "SENSE or READ LOG EXT command", NULL);
                slot = 0;
                goto out;
        } else if (command_type == AHCI_RDWR_PMULT_CMD) {
                /*
                 * An extra check on PxCI. Sometimes PxCI bits may not be
                 * cleared during hot-plug or error recovery process.
                 */
                port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, addrp->aa_port));

                if (port_cmd_issue != 0) {
                        AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
                            "ahci_claim_free_slot: there is still pending "
                            "command(s) in command list (0x%x/0x%x, PxCI %x),"
                            "so no available slot for R/W PMULT command.",
                            NON_NCQ_CMD_IN_PROGRESS(ahci_portp),
                            NCQ_CMD_IN_PROGRESS(ahci_portp),
                            port_cmd_issue);
                        return (AHCI_FAILURE);
                }

                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                    "ahci_claim_free_slot: slot 0 is allocated for "
                    "READ/WRITE PORTMULT command", NULL);
                slot = 0;
                goto out;
        }

        slot = ddi_ffs(free_slots) - 1;
        if (slot == -1) {
                AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
                    "ahci_claim_free_slot: no empty slots", NULL);
                return (AHCI_FAILURE);
        }

        /*
         * According to the AHCI spec, to allow a simple mechanism for the
         * HBA to map command list slots to queue entries, software must
         * match the tag number it uses to the slot it is placing the command
         * in. For example, if a queued command is placed in slot 5, the tag
         * for that command must be 5.
         */
        if (command_type == AHCI_NCQ_CMD) {
                ahci_portp->ahciport_pending_ncq_tags |= (0x1 << slot);
                if (AHCI_ADDR_IS_PMPORT(addrp)) {
                        ASSERT(ahci_portp->ahciport_pmult_info != NULL);
                        AHCIPORT_NCQ_PMPORT(ahci_portp) = addrp->aa_pmport;
                }
        }

        ahci_portp->ahciport_pending_tags |= (0x1 << slot);

out:
        AHCIDBG(AHCIDBG_VERBOSE, ahci_ctlp,
            "ahci_claim_free_slot: found slot: 0x%x", slot);

        return (slot);
}

/*
 * Builds the Command Table for the sata packet and delivers it to controller.
 *
 * Returns:
 *      slot number if we can obtain a slot successfully
 *      otherwise, return AHCI_FAILURE
 */
static int
ahci_deliver_satapkt(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    ahci_addr_t *addrp, sata_pkt_t *spkt)
{
        int cmd_slot;
        sata_cmd_t *scmd;
        ahci_fis_h2d_register_t *h2d_register_fisp;
        ahci_cmd_table_t *cmd_table;
        ahci_cmd_header_t *cmd_header;
        int ncookies;
        int i;
        int command_type = AHCI_NON_NCQ_CMD;
        int ncq_qdepth;
        int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
        uint8_t port, pmport;
#if AHCI_DEBUG
        uint32_t *ptr;
        uint8_t *ptr2;
#endif

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        port = addrp->aa_port;
        pmport = addrp->aa_pmport;

        spkt->satapkt_reason = SATA_PKT_BUSY;

        scmd = &spkt->satapkt_cmd;

        /* Check if the command is a NCQ command */
        if (scmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED ||
            scmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) {
                command_type = AHCI_NCQ_CMD;

                /*
                 * When NCQ is support, system software must determine the
                 * maximum tag allowed by the device and the HBA, and it
                 * must use a value not beyond of the lower bound of the two.
                 *
                 * Sata module is going to calculate the qdepth and send
                 * down to HBA driver via sata_cmd.
                 */
                ncq_qdepth = scmd->satacmd_flags.sata_max_queue_depth + 1;

                /*
                 * At the moment, the driver doesn't support the dynamic
                 * setting of the maximum ncq depth, and the value can be
                 * set either during the attach or after hot-plug insertion.
                 */
                if (ahci_portp->ahciport_max_ncq_tags == 0) {
                        ahci_portp->ahciport_max_ncq_tags = ncq_qdepth;
                        AHCIDBG(AHCIDBG_NCQ, ahci_ctlp,
                            "ahci_deliver_satapkt: port %d the max tags for "
                            "NCQ command is %d", port, ncq_qdepth);
                } else {
                        if (ncq_qdepth != ahci_portp->ahciport_max_ncq_tags) {
                                cmn_err(CE_WARN, "!ahci%d: ahci_deliver_satapkt"
                                    " port %d the max tag for NCQ command is "
                                    "requested to change from %d to %d, at the"
                                    " moment the driver doesn't support the "
                                    "dynamic change so it's going to "
                                    "still use the previous tag value",
                                    instance, port,
                                    ahci_portp->ahciport_max_ncq_tags,
                                    ncq_qdepth);
                        }
                }
        }

        /* Check if the command is an error retrieval command */
        if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
                command_type = AHCI_ERR_RETRI_CMD;

        /* Check if the command is an read/write pmult command */
        if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
                command_type = AHCI_RDWR_PMULT_CMD;

        /* Check if there is an empty command slot */
        cmd_slot = ahci_claim_free_slot(ahci_ctlp, ahci_portp,
            addrp, command_type);
        if (cmd_slot == AHCI_FAILURE) {
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "no free command slot", NULL);
                return (AHCI_FAILURE);
        }

        AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INFO, ahci_ctlp,
            "ahci_deliver_satapkt enter: cmd_reg: 0x%x, cmd_slot: 0x%x, "
            "port: %d, satapkt: 0x%p", scmd->satacmd_cmd_reg,
            cmd_slot, port, (void *)spkt);

        cmd_table = ahci_portp->ahciport_cmd_tables[cmd_slot];
        bzero((void *)cmd_table, ahci_cmd_table_size);

        /* For data transfer operations, it is the H2D Register FIS */
        h2d_register_fisp =
            &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);

        SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);

        /*
         * PMP field only make sense when target is a port multiplier or a
         * device behind a port multiplier. Otherwise should set it to 0.
         */
        if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
                SET_FIS_PMP(h2d_register_fisp, pmport);

        SET_FIS_CDMDEVCTL(h2d_register_fisp, 1);
        SET_FIS_COMMAND(h2d_register_fisp, scmd->satacmd_cmd_reg);
        SET_FIS_FEATURES(h2d_register_fisp, scmd->satacmd_features_reg);
        SET_FIS_SECTOR_COUNT(h2d_register_fisp, scmd->satacmd_sec_count_lsb);

        switch (scmd->satacmd_addr_type) {

        case 0:
                /*
                 * satacmd_addr_type will be 0 for the commands below:
                 *      ATAPI command
                 *      SATAC_IDLE_IM
                 *      SATAC_STANDBY_IM
                 *      SATAC_DOWNLOAD_MICROCODE
                 *      SATAC_FLUSH_CACHE
                 *      SATAC_SET_FEATURES
                 *      SATAC_SMART
                 *      SATAC_ID_PACKET_DEVICE
                 *      SATAC_ID_DEVICE
                 *      SATAC_READ_PORTMULT
                 *      SATAC_WRITE_PORTMULT
                 */
                /* FALLTHRU */

        case ATA_ADDR_LBA:
                /* FALLTHRU */

        case ATA_ADDR_LBA28:
                /* LBA[7:0] */
                SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);

                /* LBA[15:8] */
                SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);

                /* LBA[23:16] */
                SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);

                /* LBA [27:24] (also called dev_head) */
                SET_FIS_DEV_HEAD(h2d_register_fisp, scmd->satacmd_device_reg);

                break;

        case ATA_ADDR_LBA48:
                /* LBA[7:0] */
                SET_FIS_SECTOR(h2d_register_fisp, scmd->satacmd_lba_low_lsb);

                /* LBA[15:8] */
                SET_FIS_CYL_LOW(h2d_register_fisp, scmd->satacmd_lba_mid_lsb);

                /* LBA[23:16] */
                SET_FIS_CYL_HI(h2d_register_fisp, scmd->satacmd_lba_high_lsb);

                /* LBA [31:24] */
                SET_FIS_SECTOR_EXP(h2d_register_fisp,
                    scmd->satacmd_lba_low_msb);

                /* LBA [39:32] */
                SET_FIS_CYL_LOW_EXP(h2d_register_fisp,
                    scmd->satacmd_lba_mid_msb);

                /* LBA [47:40] */
                SET_FIS_CYL_HI_EXP(h2d_register_fisp,
                    scmd->satacmd_lba_high_msb);

                /* Set dev_head */
                SET_FIS_DEV_HEAD(h2d_register_fisp,
                    scmd->satacmd_device_reg);

                /* Set the extended sector count and features */
                SET_FIS_SECTOR_COUNT_EXP(h2d_register_fisp,
                    scmd->satacmd_sec_count_msb);
                SET_FIS_FEATURES_EXP(h2d_register_fisp,
                    scmd->satacmd_features_reg_ext);
                break;
        }

        /*
         * For NCQ command (READ/WRITE FPDMA QUEUED), sector count 7:0 is
         * filled into features field, and sector count 8:15 is filled into
         * features (exp) field. The hba driver doesn't need to anything
         * special with regard to this, since sata framework has already
         * done so.
         *
         * However the driver needs to make sure TAG is filled into sector
         * field.
         */
        if (command_type == AHCI_NCQ_CMD) {
                SET_FIS_SECTOR_COUNT(h2d_register_fisp,
                    (cmd_slot << SATA_TAG_QUEUING_SHIFT));
        }

        ncookies = scmd->satacmd_num_dma_cookies;
        AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
            "ncookies = 0x%x, ahci_dma_prdt_number = 0x%x",
            ncookies, ahci_dma_prdt_number);

        ASSERT(ncookies <= ahci_dma_prdt_number);
        ahci_portp->ahciport_prd_bytecounts[cmd_slot] = 0;

        /* *** now fill the scatter gather list ******* */
        for (i = 0; i < ncookies; i++) {
                cmd_table->ahcict_prdt[i].ahcipi_data_base_addr =
                    scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[0];
                cmd_table->ahcict_prdt[i].ahcipi_data_base_addr_upper =
                    scmd->satacmd_dma_cookie_list[i]._dmu._dmac_la[1];
                cmd_table->ahcict_prdt[i].ahcipi_descr_info =
                    scmd->satacmd_dma_cookie_list[i].dmac_size - 1;
                ahci_portp->ahciport_prd_bytecounts[cmd_slot] +=
                    scmd->satacmd_dma_cookie_list[i].dmac_size;
        }

        AHCIDBG(AHCIDBG_PRDT, ahci_ctlp,
            "ahciport_prd_bytecounts 0x%x for cmd_slot 0x%x",
            ahci_portp->ahciport_prd_bytecounts[cmd_slot], cmd_slot);

        /* The ACMD field is filled in for ATAPI command */
        if (scmd->satacmd_cmd_reg == SATAC_PACKET) {
                bcopy(scmd->satacmd_acdb, cmd_table->ahcict_atapi_cmd,
                    SATA_ATAPI_MAX_CDB_LEN);
        }

        /* Set Command Header in Command List */
        cmd_header = &ahci_portp->ahciport_cmd_list[cmd_slot];
        BZERO_DESCR_INFO(cmd_header);
        BZERO_PRD_BYTE_COUNT(cmd_header);

        /* Set the number of entries in the PRD table */
        SET_PRD_TABLE_LENGTH(cmd_header, ncookies);

        /* Set the length of the command in the CFIS area */
        SET_COMMAND_FIS_LENGTH(cmd_header, AHCI_H2D_REGISTER_FIS_LENGTH);

        /*
         * PMP field only make sense when target is a port multiplier or a
         * device behind a port multiplier. Otherwise should set it to 0.
         */
        if (AHCI_ADDR_IS_PMULT(addrp) || AHCI_ADDR_IS_PMPORT(addrp))
                SET_PORT_MULTI_PORT(cmd_header, pmport);

        AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "command data direction is "
            "sata_data_direction = 0x%x",
            scmd->satacmd_flags.sata_data_direction);

        /* Set A bit if it is an ATAPI command */
        if (scmd->satacmd_cmd_reg == SATAC_PACKET)
                SET_ATAPI(cmd_header, AHCI_CMDHEAD_ATAPI);

        /* Set W bit if data is going to the device */
        if (scmd->satacmd_flags.sata_data_direction == SATA_DIR_WRITE)
                SET_WRITE(cmd_header, AHCI_CMDHEAD_DATA_WRITE);

        /*
         * Set the prefetchable bit - this bit is only valid if the PRDTL
         * field is non-zero or the ATAPI 'A' bit is set in the command
         * header. This bit cannot be set when using native command
         * queuing commands or when using FIS-based switching with a Port
         * multiplier.
         */
        if (command_type != AHCI_NCQ_CMD)
                SET_PREFETCHABLE(cmd_header, AHCI_CMDHEAD_PREFETCHABLE);

        /*
         * Now remember the sata packet in ahciport_slot_pkts[].
         * Error retrieval command and r/w port multiplier command will
         * be stored specifically for each port.
         */
        if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
            !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
                ahci_portp->ahciport_slot_pkts[cmd_slot] = spkt;

        /*
         * Keep the timeout value
         */
        ahci_portp->ahciport_slot_timeout[cmd_slot] = spkt->satapkt_time;

        /*
         * If the intial timout is less than 1 tick, then make it longer by
         * 1 tick to avoid immediate timeout
         */
        if (ahci_portp->ahciport_slot_timeout[cmd_slot] <=
            ahci_watchdog_timeout)
                ahci_portp->ahciport_slot_timeout[cmd_slot] +=
                    ahci_watchdog_timeout;

#if AHCI_DEBUG
        if (ahci_debug_flags & AHCIDBG_ATACMD &&
            scmd->satacmd_cmd_reg != SATAC_PACKET ||
            ahci_debug_flags & AHCIDBG_ATAPICMD &&
            scmd->satacmd_cmd_reg == SATAC_PACKET) {

                /* Dump the command header and table */
                ahci_log(ahci_ctlp, CE_WARN, "\n");
                ahci_log(ahci_ctlp, CE_WARN, "Command header&table for spkt "
                    "0x%p cmd_reg 0x%x port %d", spkt,
                    scmd->satacmd_cmd_reg, port);
                ptr = (uint32_t *)cmd_header;
                ahci_log(ahci_ctlp, CE_WARN,
                    "  Command Header:%8x %8x %8x %8x",
                    ptr[0], ptr[1], ptr[2], ptr[3]);

                /* Dump the H2D register FIS */
                ptr = (uint32_t *)h2d_register_fisp;
                ahci_log(ahci_ctlp, CE_WARN,
                    "  Command FIS:   %8x %8x %8x %8x",
                    ptr[0], ptr[1], ptr[2], ptr[3]);

                /* Dump the ACMD register FIS */
                ptr2 = (uint8_t *)&(cmd_table->ahcict_atapi_cmd);
                for (i = 0; i < SATA_ATAPI_MAX_CDB_LEN/8; i++)
                        if (ahci_debug_flags & AHCIDBG_ATAPICMD)
                                ahci_log(ahci_ctlp, CE_WARN,
                                    "  ATAPI command: %2x %2x %2x %2x "
                                    "%2x %2x %2x %2x",
                                    ptr2[8 * i], ptr2[8 * i + 1],
                                    ptr2[8 * i + 2], ptr2[8 * i + 3],
                                    ptr2[8 * i + 4], ptr2[8 * i + 5],
                                    ptr2[8 * i + 6], ptr2[8 * i + 7]);

                /* Dump the PRDT */
                for (i = 0; i < ncookies; i++) {
                        ptr = (uint32_t *)&(cmd_table->ahcict_prdt[i]);
                        ahci_log(ahci_ctlp, CE_WARN,
                            "  Cookie %d:      %8x %8x %8x %8x",
                            i, ptr[0], ptr[1], ptr[2], ptr[3]);
                }
        }
#endif

        (void) ddi_dma_sync(
            ahci_portp->ahciport_cmd_tables_dma_handle[cmd_slot],
            0,
            ahci_cmd_table_size,
            DDI_DMA_SYNC_FORDEV);

        (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
            cmd_slot * sizeof (ahci_cmd_header_t),
            sizeof (ahci_cmd_header_t),
            DDI_DMA_SYNC_FORDEV);

        if ((ahci_check_dma_handle(ahci_portp->
            ahciport_cmd_tables_dma_handle[cmd_slot]) != DDI_FM_OK) ||
            ahci_check_dma_handle(ahci_portp->
            ahciport_cmd_list_dma_handle) != DDI_FM_OK) {
                ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
                    DDI_SERVICE_UNAFFECTED);
                return (AHCI_FAILURE);
        }

        /* Set the corresponding bit in the PxSACT.DS for queued command */
        if (command_type == AHCI_NCQ_CMD) {
                ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port),
                    (0x1 << cmd_slot));
        }

        /* Indicate to the HBA that a command is active. */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
            (0x1 << cmd_slot));

        AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_deliver_satapkt "
            "exit: port %d", port);

        /* Make sure the command is started by the PxSACT/PxCI */
        if (ahci_check_acc_handle(ahci_ctlp->
            ahcictl_ahci_acc_handle) != DDI_FM_OK) {
                ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
                    DDI_SERVICE_UNAFFECTED);
                return (AHCI_FAILURE);
        }

        return (cmd_slot);
}

/*
 * Called by the sata framework to abort the previously sent packet(s).
 *
 * Reset device to abort commands.
 */
static int
ahci_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag)
{
        ahci_ctl_t *ahci_ctlp;
        ahci_port_t *ahci_portp;
        uint32_t slot_status = 0;
        uint32_t aborted_tags = 0;
        uint32_t finished_tags = 0;
        uint8_t cport = spkt->satapkt_device.satadev_addr.cport;
        uint8_t port;
        int tmp_slot;
        int instance = ddi_get_instance(dip);

        ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
        port = ahci_ctlp->ahcictl_cport_to_port[cport];

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_tran_abort enter: port %d", port);

        ahci_portp = ahci_ctlp->ahcictl_ports[port];
        mutex_enter(&ahci_portp->ahciport_mutex);

        /*
         * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
         * commands are being mopped, therefore there is nothing else to do
         */
        if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                    "ahci_tran_abort: port %d is in "
                    "mopping process, so just return directly ", port);
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (SATA_SUCCESS);
        }

        /*
         * If AHCI_PORT_FLAG_RDWR_PMULT flag is set, it means a R/W PMULT
         * command is being executed so no other commands is outstanding,
         * nothing to do.
         */
        if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_RDWR_PMULT) {
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                    "ahci_tran_abort: port %d is reading/writing "
                    "port multiplier, so just return directly ", port);
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (SATA_SUCCESS);
        }

        if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
            ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
            ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
                /*
                 * In case the targer driver would send the request before
                 * sata framework can have the opportunity to process those
                 * event reports.
                 */
                spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
                spkt->satapkt_device.satadev_state =
                    ahci_portp->ahciport_port_state;
                ahci_update_sata_registers(ahci_ctlp, port,
                    &spkt->satapkt_device);
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_tran_abort returning SATA_FAILURE while "
                    "port in FAILED/SHUTDOWN/PWROFF state: "
                    "port: %d", port);
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (SATA_FAILURE);
        }

        if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
                /*
                 * ahci_intr_phyrdy_change() may have rendered it to
                 * AHCI_PORT_TYPE_NODEV.
                 */
                spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
                spkt->satapkt_device.satadev_type = SATA_DTYPE_NONE;
                spkt->satapkt_device.satadev_state =
                    ahci_portp->ahciport_port_state;
                ahci_update_sata_registers(ahci_ctlp, port,
                    &spkt->satapkt_device);
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_tran_abort returning SATA_FAILURE while "
                    "no device attached: port: %d", port);
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (SATA_FAILURE);
        }

        if (flag == SATA_ABORT_ALL_PACKETS) {
                if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
                        aborted_tags = ahci_portp->ahciport_pending_tags;
                else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
                        aborted_tags = ahci_portp->ahciport_pending_ncq_tags;

                cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort all packets",
                    instance, port);
        } else {
                aborted_tags = 0xffffffff;
                /*
                 * Aborting one specific packet, first search the
                 * ahciport_slot_pkts[] list for matching spkt.
                 */
                for (tmp_slot = 0;
                    tmp_slot < ahci_ctlp->ahcictl_num_cmd_slots; tmp_slot++) {
                        if (ahci_portp->ahciport_slot_pkts[tmp_slot] == spkt) {
                                aborted_tags = (0x1 << tmp_slot);
                                break;
                        }
                }

                if (aborted_tags == 0xffffffff) {
                        /* request packet is not on the pending list */
                        AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                            "Cannot find the aborting pkt 0x%p on the "
                            "pending list", (void *)spkt);
                        ahci_update_sata_registers(ahci_ctlp, port,
                            &spkt->satapkt_device);
                        mutex_exit(&ahci_portp->ahciport_mutex);
                        return (SATA_FAILURE);
                }
                cmn_err(CE_NOTE, "!ahci%d: ahci port %d abort satapkt 0x%p",
                    instance, port, (void *)spkt);
        }

        if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
                slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
        else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
                slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));

        ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
        ahci_portp->ahciport_mop_in_progress++;

        /*
         * To abort the packet(s), first we are trying to clear PxCMD.ST
         * to stop the port, and if the port can be stopped
         * successfully with PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0',
         * then we just send back the aborted packet(s) with ABORTED flag
         * and then restart the port by setting PxCMD.ST and PxCMD.FRE.
         * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then we
         * perform a COMRESET.
         */
        (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
            ahci_portp, port, 0, NULL);

        /*
         * Compute which have finished and which need to be retried.
         *
         * The finished tags are ahciport_pending_tags/ahciport_pending_ncq_tags
         * minus the slot_status. The aborted_tags has to be deducted by
         * finished_tags since we can't possibly abort a tag which had finished
         * already.
         */
        if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
                finished_tags = ahci_portp->ahciport_pending_tags &
                    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
        else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
                finished_tags = ahci_portp->ahciport_pending_ncq_tags &
                    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);

        aborted_tags &= ~finished_tags;

        ahci_mop_commands(ahci_ctlp,
            ahci_portp,
            slot_status,
            0, /* failed tags */
            0, /* timeout tags */
            aborted_tags,
            0); /* reset tags */

        ahci_update_sata_registers(ahci_ctlp, port, &spkt->satapkt_device);
        mutex_exit(&ahci_portp->ahciport_mutex);

        return (SATA_SUCCESS);
}

/*
 * Used to do device reset and reject all the pending packets on a device
 * during the reset operation.
 *
 * NOTE: ONLY called by ahci_tran_reset_dport
 */
static int
ahci_reset_device_reject_pkts(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp, ahci_addr_t *addrp)
{
        uint32_t slot_status = 0;
        uint32_t reset_tags = 0;
        uint32_t finished_tags = 0;
        uint8_t port = addrp->aa_port;
        sata_device_t sdevice;
        int ret;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_reset_device_reject_pkts on port: %d", port);

        /*
         * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
         * commands are being mopped, therefore there is nothing else to do
         */
        if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_reset_device_reject_pkts: port %d is in "
                    "mopping process, so return directly ", port);
                return (SATA_SUCCESS);
        }

        if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
                reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
        } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
                reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
        }

        if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
            != AHCI_SUCCESS) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "Try to do a port reset after software "
                    "reset failed", port);
                ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);
                if (ret != AHCI_SUCCESS) {
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_reset_device_reject_pkts: port %d "
                            "failed", port);
                        return (SATA_FAILURE);
                }
        }
        /* Set the reset in progress flag */
        ahci_portp->ahciport_reset_in_progress = 1;

        ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
        ahci_portp->ahciport_mop_in_progress++;

        /* Indicate to the framework that a reset has happened */
        bzero((void *)&sdevice, sizeof (sata_device_t));
        sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
        sdevice.satadev_addr.pmport = 0;
        sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
        sdevice.satadev_state = SATA_DSTATE_RESET |
            SATA_DSTATE_PWR_ACTIVE;
        mutex_exit(&ahci_portp->ahciport_mutex);
        sata_hba_event_notify(
            ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
            &sdevice,
            SATA_EVNT_DEVICE_RESET);
        mutex_enter(&ahci_portp->ahciport_mutex);

        AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
            "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);

        /* Next try to mop the pending commands */
        if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
                finished_tags = ahci_portp->ahciport_pending_tags &
                    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
        else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
                finished_tags = ahci_portp->ahciport_pending_ncq_tags &
                    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);

        reset_tags &= ~finished_tags;

        ahci_mop_commands(ahci_ctlp,
            ahci_portp,
            slot_status,
            0, /* failed tags */
            0, /* timeout tags */
            0, /* aborted tags */
            reset_tags); /* reset tags */

        return (SATA_SUCCESS);
}

/*
 * Used to do device reset and reject all the pending packets on a device
 * during the reset operation.
 *
 * NOTE: ONLY called by ahci_tran_reset_dport
 */
static int
ahci_reset_pmdevice_reject_pkts(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp, ahci_addr_t *addrp)
{
        uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
        uint8_t port = addrp->aa_port;
        uint8_t pmport = addrp->aa_pmport;
        sata_device_t sdevice;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_PMULT, ahci_ctlp,
            "ahci_reset_pmdevice_reject_pkts at port %d:%d", port, pmport);

        if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_reset_pmdevice_reject_pkts: port %d is in "
                    "mopping process, so return directly ", port);
                return (SATA_SUCCESS);
        }

        /* Checking for outstanding commands */
        if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
                reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
        } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
                reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
        }

        /* Issue SOFTWARE reset command. */
        if (ahci_software_reset(ahci_ctlp, ahci_portp, addrp)
            != AHCI_SUCCESS) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "Try to do a port reset after software "
                    "reset failed", port);
                return (SATA_FAILURE);
        }

        /* Set the reset in progress flag */
        ahci_portp->ahciport_reset_in_progress = 1;

        ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
        ahci_portp->ahciport_mop_in_progress++;

        /* Indicate to the framework that a reset has happened */
        bzero((void *)&sdevice, sizeof (sata_device_t));
        sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
        sdevice.satadev_addr.pmport = pmport;
        if (AHCI_ADDR_IS_PMULT(addrp))
                sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
        else
                sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT;
        sdevice.satadev_state = SATA_DSTATE_RESET |
            SATA_DSTATE_PWR_ACTIVE;
        mutex_exit(&ahci_portp->ahciport_mutex);
        sata_hba_event_notify(
            ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
            &sdevice,
            SATA_EVNT_DEVICE_RESET);
        mutex_enter(&ahci_portp->ahciport_mutex);

        AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
            "port %d:%d sending event up: SATA_EVNT_DEVICE_RESET",
            port, pmport);

        /* Next try to mop the pending commands */
        if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
                finished_tags = ahci_portp->ahciport_pending_tags &
                    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
        else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
                finished_tags = ahci_portp->ahciport_pending_ncq_tags &
                    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
        reset_tags &= ~finished_tags;

        AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
            "reset_tags = %x, finished_tags = %x, slot_status = %x",
            reset_tags, finished_tags, slot_status);

        /*
         * NOTE: Because PxCI be only erased by unset PxCMD.ST bit, so even we
         * try to reset a single device behind a port multiplier will
         * terminate all the commands on that HBA port. We need mop these
         * commands as well.
         */
        ahci_mop_commands(ahci_ctlp,
            ahci_portp,
            slot_status,
            0, /* failed tags */
            0, /* timeout tags */
            0, /* aborted tags */
            reset_tags); /* reset tags */

        return (SATA_SUCCESS);
}

/*
 * Used to do port reset and reject all the pending packets on a port during
 * the reset operation.
 */
static int
ahci_reset_port_reject_pkts(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp, ahci_addr_t *addrp)
{
        uint32_t slot_status = 0;
        uint32_t reset_tags = 0;
        uint32_t finished_tags = 0;
        uint8_t port = addrp->aa_port;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_reset_port_reject_pkts at port: %d", port);

        /*
         * If AHCI_PORT_FLAG_MOPPING flag is set, it means all the pending
         * commands are being mopped, therefore there is nothing else to do
         */
        if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_reset_port_reject_pkts: port %d is in "
                    "mopping process, so return directly ", port);
                return (SATA_SUCCESS);
        }

        ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
        ahci_portp->ahciport_mop_in_progress++;

        if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
                reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
        } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
                reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
        }

        if (ahci_restart_port_wait_till_ready(ahci_ctlp,
            ahci_portp, port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
            NULL) != AHCI_SUCCESS) {

                /* Clear mop flag */
                ahci_portp->ahciport_mop_in_progress--;
                if (ahci_portp->ahciport_mop_in_progress == 0)
                        ahci_portp->ahciport_flags &=
                            ~AHCI_PORT_FLAG_MOPPING;
                return (SATA_FAILURE);
        }

        if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
                finished_tags = ahci_portp->ahciport_pending_tags &
                    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
        else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
                finished_tags = ahci_portp->ahciport_pending_ncq_tags &
                    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);

        reset_tags &= ~finished_tags;

        ahci_mop_commands(ahci_ctlp,
            ahci_portp,
            slot_status,
            0, /* failed tags */
            0, /* timeout tags */
            0, /* aborted tags */
            reset_tags); /* reset tags */

        return (SATA_SUCCESS);
}

/*
 * Used to do hba reset and reject all the pending packets on all ports
 * during the reset operation.
 */
static int
ahci_reset_hba_reject_pkts(ahci_ctl_t *ahci_ctlp)
{
        ahci_port_t *ahci_portp;
        uint32_t slot_status[AHCI_MAX_PORTS];
        uint32_t reset_tags[AHCI_MAX_PORTS];
        uint32_t finished_tags[AHCI_MAX_PORTS];
        int port;
        int ret = SATA_SUCCESS;

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_reset_hba_reject_pkts enter", NULL);

        bzero(slot_status, sizeof (slot_status));
        bzero(reset_tags, sizeof (reset_tags));
        bzero(finished_tags, sizeof (finished_tags));

        for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
                if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
                        continue;
                }

                ahci_portp = ahci_ctlp->ahcictl_ports[port];

                mutex_enter(&ahci_portp->ahciport_mutex);
                ahci_portp->ahciport_reset_in_progress = 1;
                if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                        slot_status[port] = ddi_get32(
                            ahci_ctlp->ahcictl_ahci_acc_handle,
                            (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
                        reset_tags[port] = slot_status[port] &
                            AHCI_SLOT_MASK(ahci_ctlp);
                        AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                            "port %d: reset_tags = 0x%x pending_tags = 0x%x",
                            port, reset_tags[port],
                            ahci_portp->ahciport_pending_tags);
                } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                        slot_status[port] = ddi_get32(
                            ahci_ctlp->ahcictl_ahci_acc_handle,
                            (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
                        reset_tags[port] = slot_status[port] &
                            AHCI_NCQ_SLOT_MASK(ahci_portp);
                        AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                            "port %d: reset_tags = 0x%x pending_tags = 0x%x",
                            port, reset_tags[port],
                            ahci_portp->ahciport_pending_tags);
                }
                mutex_exit(&ahci_portp->ahciport_mutex);
        }

        if (ahci_hba_reset(ahci_ctlp) != AHCI_SUCCESS) {
                ret = SATA_FAILURE;
        }

        for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
                if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
                        continue;
                }

                ahci_portp = ahci_ctlp->ahcictl_ports[port];

                mutex_enter(&ahci_portp->ahciport_mutex);
                /*
                 * To prevent recursive enter to ahci_mop_commands, we need
                 * check AHCI_PORT_FLAG_MOPPING flag.
                 */
                if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_reset_hba_reject_pkts: port %d is in "
                            "mopping process, so return directly ", port);
                        mutex_exit(&ahci_portp->ahciport_mutex);
                        continue;
                }

                ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
                ahci_portp->ahciport_mop_in_progress++;

                if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
                        finished_tags[port]  =
                            ahci_portp->ahciport_pending_tags &
                            ~slot_status[port] & AHCI_SLOT_MASK(ahci_ctlp);
                else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
                        finished_tags[port] =
                            ahci_portp->ahciport_pending_ncq_tags &
                            ~slot_status[port] & AHCI_NCQ_SLOT_MASK(ahci_portp);

                reset_tags[port] &= ~finished_tags[port];

                ahci_mop_commands(ahci_ctlp,
                    ahci_portp,
                    slot_status[port],
                    0, /* failed tags */
                    0, /* timeout tags */
                    0, /* aborted tags */
                    reset_tags[port]); /* reset tags */
                mutex_exit(&ahci_portp->ahciport_mutex);
        }

        return (ret);
}

/*
 * Called by sata framework to reset a port(s) or device.
 */
static int
ahci_tran_reset_dport(dev_info_t *dip, sata_device_t *sd)
{
        ahci_ctl_t *ahci_ctlp;
        ahci_port_t *ahci_portp;
        ahci_addr_t addr;
        uint8_t cport = sd->satadev_addr.cport;
        uint8_t pmport = sd->satadev_addr.pmport;
        uint8_t port;
        int ret = SATA_SUCCESS;
        int instance = ddi_get_instance(dip);

        ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
        port = ahci_ctlp->ahcictl_cport_to_port[cport];
        ahci_portp = ahci_ctlp->ahcictl_ports[port];

        ahci_get_ahci_addr(ahci_ctlp, sd, &addr);

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_tran_reset_dport enter: cport %d", cport);

        switch (sd->satadev_addr.qual) {
        case SATA_ADDR_PMPORT:
                /*
                 * If we want to issue a COMRESET on a pmport, we need to
                 * reject the outstanding commands on that pmport. According
                 * to AHCI spec, PxCI register could only be cleared by
                 * clearing PxCMD.ST, which will halt the controller port - as
                 * well as other pmports.
                 *
                 * Therefore we directly reset the controller port for
                 * simplicity. ahci_tran_probe_port() will handle reset stuff
                 * like initializing the given pmport.
                 */
                /* FALLTHRU */
        case SATA_ADDR_CPORT:
                /* Port reset */
                ahci_portp = ahci_ctlp->ahcictl_ports[port];
                cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
                    "port %d reset port", instance, port);

                mutex_enter(&ahci_portp->ahciport_mutex);
                ret = ahci_reset_port_reject_pkts(ahci_ctlp, ahci_portp, &addr);
                mutex_exit(&ahci_portp->ahciport_mutex);

                break;

        case SATA_ADDR_DPMPORT:
                cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
                    "port %d:%d reset device", instance, port, pmport);
                /* FALLTHRU */
        case SATA_ADDR_DCPORT:
                /* Device reset */
                if (sd->satadev_addr.qual == SATA_ADDR_DCPORT)
                        cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
                            "port %d reset device", instance, port);

                mutex_enter(&ahci_portp->ahciport_mutex);
                /*
                 * software reset request must be sent to SATA_PMULT_HOSTPORT
                 * if target is a port multiplier:
                 */
                if (sd->satadev_addr.qual == SATA_ADDR_DCPORT &&
                    ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT)
                        AHCI_ADDR_SET_PMULT(&addr, port);

                if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED |
                    ahci_portp->ahciport_port_state & SATA_PSTATE_SHUTDOWN |
                    ahci_portp->ahciport_port_state & SATA_PSTATE_PWROFF) {
                        /*
                         * In case the targer driver would send the request
                         * before sata framework can have the opportunity to
                         * process those event reports.
                         */
                        sd->satadev_state = ahci_portp->ahciport_port_state;
                        ahci_update_sata_registers(ahci_ctlp, port, sd);
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_tran_reset_dport returning SATA_FAILURE "
                            "while port in FAILED/SHUTDOWN/PWROFF state: "
                            "port: %d", port);
                        mutex_exit(&ahci_portp->ahciport_mutex);
                        ret = SATA_FAILURE;
                        break;
                }

                if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr) ==
                    SATA_DTYPE_NONE) {
                        /*
                         * ahci_intr_phyrdy_change() may have rendered it to
                         * AHCI_PORT_TYPE_NODEV.
                         */
                        sd->satadev_type = SATA_DTYPE_NONE;
                        sd->satadev_state = AHCIPORT_GET_STATE(ahci_portp,
                            &addr);
                        ahci_update_sata_registers(ahci_ctlp, port, sd);
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_tran_reset_dport returning SATA_FAILURE "
                            "while no device attached: port: %d", port);
                        mutex_exit(&ahci_portp->ahciport_mutex);
                        ret = SATA_FAILURE;
                        break;
                }

                if (AHCI_ADDR_IS_PORT(&addr)) {
                        ret = ahci_reset_device_reject_pkts(ahci_ctlp,
                            ahci_portp, &addr);
                } else {
                        ret = ahci_reset_pmdevice_reject_pkts(ahci_ctlp,
                            ahci_portp, &addr);
                }

                mutex_exit(&ahci_portp->ahciport_mutex);
                break;

        case SATA_ADDR_CNTRL:
                /* Reset the whole controller */
                cmn_err(CE_NOTE, "!ahci%d: ahci_tran_reset_dport "
                    "reset the whole hba", instance);
                ret = ahci_reset_hba_reject_pkts(ahci_ctlp);
                break;

        default:
                ret = SATA_FAILURE;
        }

        return (ret);
}

/*
 * Called by sata framework to activate a port as part of hotplug.
 * (cfgadm -c connect satax/y)
 * Support port multiplier.
 */
static int
ahci_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev)
{
        ahci_ctl_t *ahci_ctlp;
        ahci_port_t *ahci_portp;
        ahci_addr_t addr;
        uint8_t cport = satadev->satadev_addr.cport;
        uint8_t pmport = satadev->satadev_addr.pmport;
        uint8_t port;
        int instance = ddi_get_instance(dip);

        ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
        port = ahci_ctlp->ahcictl_cport_to_port[cport];

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_tran_hotplug_port_activate enter: cport %d", cport);

        ahci_portp = ahci_ctlp->ahcictl_ports[port];

        mutex_enter(&ahci_portp->ahciport_mutex);
        ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
        ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));

        if (AHCI_ADDR_IS_PORT(&addr)) {
                cmn_err(CE_NOTE, "!ahci%d: ahci port %d is activated",
                    instance, port);

                /* Enable the interrupts on the port */
                ahci_enable_port_intrs(ahci_ctlp, port);

                /*
                 * Reset the port so that the PHY communication would be
                 * re-established.  But this reset is an internal operation
                 * and the sata module doesn't need to know about it.
                 * Moreover, the port with a device attached will be started
                 * too.
                 */
                (void) ahci_restart_port_wait_till_ready(ahci_ctlp,
                    ahci_portp, port,
                    AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP,
                    NULL);

                /*
                 * Need to check the link status and device status of the port
                 * and consider raising power if the port was in D3 state
                 */
                ahci_portp->ahciport_port_state |= SATA_PSTATE_PWRON;
                ahci_portp->ahciport_port_state &= ~SATA_PSTATE_PWROFF;
                ahci_portp->ahciport_port_state &= ~SATA_PSTATE_SHUTDOWN;
        } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
                cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is activated",
                    instance, port, pmport);
                /* AHCI_ADDR_PMPORT */
                AHCIPORT_PMSTATE(ahci_portp, &addr) |= SATA_PSTATE_PWRON;
                AHCIPORT_PMSTATE(ahci_portp, &addr) &=
                    ~(SATA_PSTATE_PWROFF|SATA_PSTATE_SHUTDOWN);
        }

        satadev->satadev_state = ahci_portp->ahciport_port_state;

        ahci_update_sata_registers(ahci_ctlp, port, satadev);

        mutex_exit(&ahci_portp->ahciport_mutex);
        return (SATA_SUCCESS);
}

/*
 * Called by sata framework to deactivate a port as part of hotplug.
 * (cfgadm -c disconnect satax/y)
 * Support port multiplier.
 */
static int
ahci_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev)
{
        ahci_ctl_t *ahci_ctlp;
        ahci_port_t *ahci_portp;
        ahci_addr_t addr;
        uint8_t cport = satadev->satadev_addr.cport;
        uint8_t pmport = satadev->satadev_addr.pmport;
        uint8_t port;
        uint32_t port_scontrol;
        int instance = ddi_get_instance(dip);

        ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);
        port = ahci_ctlp->ahcictl_cport_to_port[cport];

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_tran_hotplug_port_deactivate enter: cport %d", cport);

        ahci_portp = ahci_ctlp->ahcictl_ports[port];
        mutex_enter(&ahci_portp->ahciport_mutex);
        ahci_get_ahci_addr(ahci_ctlp, satadev, &addr);
        ASSERT(AHCI_ADDR_IS_PORT(&addr) || AHCI_ADDR_IS_PMPORT(&addr));

        if (AHCI_ADDR_IS_PORT(&addr)) {
                cmn_err(CE_NOTE, "!ahci%d: ahci port %d is deactivated",
                    instance, port);

                /* Disable the interrupts on the port */
                ahci_disable_port_intrs(ahci_ctlp, port);

                if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {

                        /* First to abort all the pending commands */
                        ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);

                        /* Then stop the port */
                        (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
                            ahci_portp, port);
                }

                /* Next put the PHY offline */
                port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
                SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_DISABLE);
                ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle, (uint32_t *)
                    AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);
        } else if (AHCI_ADDR_IS_PMPORT(&addr)) {
                cmn_err(CE_NOTE, "!ahci%d: ahci port %d:%d is deactivated",
                    instance, port, pmport);

                ahci_disable_port_intrs(ahci_ctlp, port);
                if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &addr)
                    != SATA_DTYPE_NONE)
                        ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);

                /* Re-enable the interrupts for the other pmports */
                ahci_enable_port_intrs(ahci_ctlp, port);
        }

        /* Update port state */
        AHCIPORT_SET_STATE(ahci_portp, &addr, SATA_PSTATE_SHUTDOWN);
        satadev->satadev_state = SATA_PSTATE_SHUTDOWN;

        ahci_update_sata_registers(ahci_ctlp, port, satadev);

        mutex_exit(&ahci_portp->ahciport_mutex);
        return (SATA_SUCCESS);
}

/*
 * To be used to mark all the outstanding pkts with SATA_PKT_ABORTED
 * when a device is unplugged or a port is deactivated.
 */
static void
ahci_reject_all_abort_pkts(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp, uint8_t port)
{
        uint32_t slot_status = 0;
        uint32_t abort_tags = 0;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
            "ahci_reject_all_abort_pkts at port: %d", port);

        /* Read/write port multiplier command takes highest priority */
        if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
                slot_status = 0x1;
                abort_tags = 0x1;
                goto out;
        }

        /*
         * When AHCI_PORT_FLAG_MOPPING is set, we need to check whether a
         * REQUEST SENSE command or READ LOG EXT command is delivered to HBA
         * to get the error data, if yes when the device is removed, the
         * command needs to be aborted too.
         */
        if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) {
                if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
                        slot_status = 0x1;
                        abort_tags = 0x1;
                        goto out;
                } else {
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_reject_all_abort_pkts return directly "
                            "port %d no needs to reject any outstanding "
                            "commands", port);
                        return;
                }
        }

        if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
                abort_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
        } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
                abort_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
        }

out:
        /* No need to do mop when there is no outstanding commands */
        if (slot_status != 0) {
                ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
                ahci_portp->ahciport_mop_in_progress++;

                ahci_mop_commands(ahci_ctlp,
                    ahci_portp,
                    slot_status,
                    0, /* failed tags */
                    0, /* timeout tags */
                    abort_tags, /* aborting tags */
                    0); /* reset tags */
        }
}

#if defined(__lock_lint)
static int
ahci_selftest(dev_info_t *dip, sata_device_t *device)
{
        return (SATA_SUCCESS);
}
#endif

/*
 * Initialize fma capabilities and register with IO fault services.
 */
static void
ahci_fm_init(ahci_ctl_t *ahci_ctlp)
{
        /*
         * Need to change iblock to priority for new MSI intr
         */
        ddi_iblock_cookie_t fm_ibc;

        ahci_ctlp->ahcictl_fm_cap = ddi_getprop(DDI_DEV_T_ANY,
            ahci_ctlp->ahcictl_dip,
            DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
            DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
            DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);

        /* Only register with IO Fault Services if we have some capability */
        if (ahci_ctlp->ahcictl_fm_cap) {
                /* Adjust access and dma attributes for FMA */
                accattr.devacc_attr_access = DDI_FLAGERR_ACC;
                buffer_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
                rcvd_fis_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
                cmd_list_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
                cmd_table_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;

                /*
                 * Register capabilities with IO Fault Services.
                 * ahcictl_fm_cap will be updated to indicate
                 * capabilities actually supported (not requested.)
                 */
                ddi_fm_init(ahci_ctlp->ahcictl_dip,
                    &ahci_ctlp->ahcictl_fm_cap, &fm_ibc);

                if (ahci_ctlp->ahcictl_fm_cap == DDI_FM_NOT_CAPABLE) {
                        cmn_err(CE_WARN, "!ahci%d: fma init failed.",
                            ddi_get_instance(ahci_ctlp->ahcictl_dip));
                        return;
                }
                /*
                 * Initialize pci ereport capabilities if ereport
                 * capable (should always be.)
                 */
                if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
                    DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
                        pci_ereport_setup(ahci_ctlp->ahcictl_dip);
                }

                /*
                 * Register error callback if error callback capable.
                 */
                if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
                        ddi_fm_handler_register(ahci_ctlp->ahcictl_dip,
                            ahci_fm_error_cb, (void *) ahci_ctlp);
                }

                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "ahci_fm_fini: fma enabled.", NULL);
        }
}

/*
 * Releases fma capabilities and un-registers with IO fault services.
 */
static void
ahci_fm_fini(ahci_ctl_t *ahci_ctlp)
{
        /* Only unregister FMA capabilities if registered */
        if (ahci_ctlp->ahcictl_fm_cap) {
                /*
                 * Un-register error callback if error callback capable.
                 */
                if (DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
                        ddi_fm_handler_unregister(ahci_ctlp->ahcictl_dip);
                }

                /*
                 * Release any resources allocated by pci_ereport_setup()
                 */
                if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap) ||
                    DDI_FM_ERRCB_CAP(ahci_ctlp->ahcictl_fm_cap)) {
                        pci_ereport_teardown(ahci_ctlp->ahcictl_dip);
                }

                /* Unregister from IO Fault Services */
                ddi_fm_fini(ahci_ctlp->ahcictl_dip);

                /* Adjust access and dma attributes for FMA */
                accattr.devacc_attr_access = DDI_DEFAULT_ACC;
                buffer_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
                rcvd_fis_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
                cmd_list_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
                cmd_table_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;

                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "ahci_fm_fini: fma disabled.", NULL);
        }
}

/*ARGSUSED*/
static int
ahci_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
{
        /*
         * as the driver can always deal with an error in any dma or
         * access handle, we can just return the fme_status value.
         */
        pci_ereport_post(dip, err, NULL);
        return (err->fme_status);
}

int
ahci_check_acc_handle(ddi_acc_handle_t handle)
{
        ddi_fm_error_t de;

        ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
        return (de.fme_status);
}

int
ahci_check_dma_handle(ddi_dma_handle_t handle)
{
        ddi_fm_error_t de;

        ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
        return (de.fme_status);
}

/*
 * Generate an ereport
 */
void
ahci_fm_ereport(ahci_ctl_t *ahci_ctlp, char *detail)
{
        uint64_t ena;
        char buf[FM_MAX_CLASS];

        (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
        ena = fm_ena_generate(0, FM_ENA_FMT1);
        if (DDI_FM_EREPORT_CAP(ahci_ctlp->ahcictl_fm_cap)) {
                ddi_fm_ereport_post(ahci_ctlp->ahcictl_dip, buf, ena,
                    DDI_NOSLEEP, FM_VERSION, DATA_TYPE_UINT8,
                    FM_EREPORT_VERSION, NULL);
        }
}

/*
 * Check if all handles are correctly allocated.
 */
static int
ahci_check_all_handle(ahci_ctl_t *ahci_ctlp)
{
        int port;

        if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
                return (DDI_FAILURE);
        }

        for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
                ahci_port_t *ahci_portp;

                if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
                        continue;

                ahci_portp = ahci_ctlp->ahcictl_ports[port];

                mutex_enter(&ahci_portp->ahciport_mutex);
                if (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS) {
                        mutex_exit(&ahci_portp->ahciport_mutex);
                        return (DDI_FAILURE);
                }
                mutex_exit(&ahci_portp->ahciport_mutex);
        }

        return (DDI_SUCCESS);
}

/*
 * Check the access handles for the controller. Note that
 * ahcictl_pci_conf_handle is only used in attach process.
 */
static int
ahci_check_ctl_handle(ahci_ctl_t *ahci_ctlp)
{
        if ((ahci_check_acc_handle(ahci_ctlp->
            ahcictl_pci_conf_handle) != DDI_FM_OK) ||
            (ahci_check_acc_handle(ahci_ctlp->
            ahcictl_ahci_acc_handle) != DDI_FM_OK)) {
                return (DDI_FAILURE);
        }
        return (DDI_SUCCESS);
}

/*
 * Check the DMA handles and the access handles of a controller port.
 */
static int
ahci_check_port_handle(ahci_ctl_t *ahci_ctlp, int port)
{
        ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];
        int slot;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        if ((ahci_check_dma_handle(ahci_portp->
            ahciport_rcvd_fis_dma_handle) != DDI_FM_OK) ||
            (ahci_check_dma_handle(ahci_portp->
            ahciport_cmd_list_dma_handle) != DDI_FM_OK) ||
            (ahci_check_acc_handle(ahci_portp->
            ahciport_rcvd_fis_acc_handle) != DDI_FM_OK) ||
            (ahci_check_acc_handle(ahci_portp->
            ahciport_cmd_list_acc_handle) != DDI_FM_OK)) {
                return (DDI_FAILURE);
        }
        for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
                if (ahci_check_slot_handle(ahci_portp, slot)
                    != DDI_SUCCESS) {
                        return (DDI_FAILURE);
                }
        }
        return (DDI_SUCCESS);
}

/*
 * Check the DMA handles and the access handles of a cmd table slot.
 */
static int
ahci_check_slot_handle(ahci_port_t *ahci_portp, int slot)
{
        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        if ((ahci_check_acc_handle(ahci_portp->
            ahciport_cmd_tables_acc_handle[slot]) != DDI_FM_OK) ||
            (ahci_check_dma_handle(ahci_portp->
            ahciport_cmd_tables_dma_handle[slot]) != DDI_FM_OK)) {
                return (DDI_FAILURE);
        }
        return (DDI_SUCCESS);
}

/*
 * Allocate the ports structure, only called by ahci_attach
 */
static int
ahci_alloc_ports_state(ahci_ctl_t *ahci_ctlp)
{
        int port, cport = 0;

        AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_alloc_ports_state enter", NULL);

        mutex_enter(&ahci_ctlp->ahcictl_mutex);

        /* Allocate structures only for the implemented ports */
        for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
                if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
                        AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                            "hba port %d not implemented", port);
                        continue;
                }

                ahci_ctlp->ahcictl_cport_to_port[cport] = (uint8_t)port;
                ahci_ctlp->ahcictl_port_to_cport[port] =
                    (uint8_t)cport++;

                if (ahci_alloc_port_state(ahci_ctlp, port) != AHCI_SUCCESS) {
                        goto err_out;
                }
        }

        mutex_exit(&ahci_ctlp->ahcictl_mutex);
        return (AHCI_SUCCESS);

err_out:
        for (port--; port >= 0; port--) {
                if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
                        ahci_dealloc_port_state(ahci_ctlp, port);
                }
        }

        mutex_exit(&ahci_ctlp->ahcictl_mutex);
        return (AHCI_FAILURE);
}

/*
 * Reverse of ahci_alloc_ports_state(), only called by ahci_detach
 */
static void
ahci_dealloc_ports_state(ahci_ctl_t *ahci_ctlp)
{
        int port;

        mutex_enter(&ahci_ctlp->ahcictl_mutex);
        for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
                /* if this port is implemented by the HBA */
                if (AHCI_PORT_IMPLEMENTED(ahci_ctlp, port))
                        ahci_dealloc_port_state(ahci_ctlp, port);
        }
        mutex_exit(&ahci_ctlp->ahcictl_mutex);
}

/*
 * Drain the taskq.
 */
static void
ahci_drain_ports_taskq(ahci_ctl_t *ahci_ctlp)
{
        ahci_port_t *ahci_portp;
        int port;

        for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
                if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
                        continue;
                }

                ahci_portp = ahci_ctlp->ahcictl_ports[port];

                mutex_enter(&ahci_portp->ahciport_mutex);
                ddi_taskq_wait(ahci_portp->ahciport_event_taskq);
                mutex_exit(&ahci_portp->ahciport_mutex);
        }
}

/*
 * Initialize the controller and all ports. And then try to start the ports
 * if there are devices attached.
 *
 * This routine can be called from three seperate cases: DDI_ATTACH,
 * PM_LEVEL_D0 and DDI_RESUME. The DDI_ATTACH case is different from
 * other two cases; device signature probing are attempted only during
 * DDI_ATTACH case.
 */
static int
ahci_initialize_controller(ahci_ctl_t *ahci_ctlp)
{
        ahci_port_t *ahci_portp;
        ahci_addr_t addr;
        int port;

        AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_initialize_controller enter", NULL);

        /* Disable the whole controller interrupts */
        mutex_enter(&ahci_ctlp->ahcictl_mutex);
        ahci_disable_all_intrs(ahci_ctlp);
        mutex_exit(&ahci_ctlp->ahcictl_mutex);

        /* Initialize the implemented ports and structures */
        for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
                if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
                        continue;
                }

                ahci_portp = ahci_ctlp->ahcictl_ports[port];
                mutex_enter(&ahci_portp->ahciport_mutex);

                /*
                 * Ensure that the controller is not in the running state
                 * by checking every implemented port's PxCMD register
                 */
                AHCI_ADDR_SET_PORT(&addr, (uint8_t)port);

                if (ahci_initialize_port(ahci_ctlp, ahci_portp, &addr)
                    != AHCI_SUCCESS) {
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_initialize_controller: failed to "
                            "initialize port %d", port);
                        /*
                         * Set the port state to SATA_PSTATE_FAILED if
                         * failed to initialize it.
                         */
                        ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
                }

                mutex_exit(&ahci_portp->ahciport_mutex);
        }

        /* Enable the whole controller interrupts */
        mutex_enter(&ahci_ctlp->ahcictl_mutex);
        ahci_enable_all_intrs(ahci_ctlp);
        mutex_exit(&ahci_ctlp->ahcictl_mutex);

        return (AHCI_SUCCESS);
}

/*
 * Reverse of ahci_initialize_controller()
 *
 * We only need to stop the ports and disable the interrupt.
 */
static void
ahci_uninitialize_controller(ahci_ctl_t *ahci_ctlp)
{
        ahci_port_t *ahci_portp;
        int port;

        AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
            "ahci_uninitialize_controller enter", NULL);

        /* disable all the interrupts. */
        mutex_enter(&ahci_ctlp->ahcictl_mutex);
        ahci_disable_all_intrs(ahci_ctlp);
        mutex_exit(&ahci_ctlp->ahcictl_mutex);

        for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
                if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
                        continue;
                }

                ahci_portp = ahci_ctlp->ahcictl_ports[port];

                /* Stop the port by clearing PxCMD.ST */
                mutex_enter(&ahci_portp->ahciport_mutex);

                /*
                 * Here we must disable the port interrupt because
                 * ahci_disable_all_intrs only clear GHC.IE, and IS
                 * register will be still set if PxIE is enabled.
                 * When ahci shares one IRQ with other drivers, the
                 * intr handler may claim the intr mistakenly.
                 */
                ahci_disable_port_intrs(ahci_ctlp, port);
                (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
                    ahci_portp, port);
                mutex_exit(&ahci_portp->ahciport_mutex);
        }
}

/*
 * ahci_alloc_pmult()
 * 1. Setting HBA port registers which are necessary for a port multiplier.
 *    (Set PxCMD.PMA while PxCMD.ST is '0')
 * 2. Allocate ahci_pmult_info structure.
 *
 * NOTE: Must stop port before the function is called.
 */
static void
ahci_alloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
{
        uint32_t port_cmd_status;
        uint8_t port = ahci_portp->ahciport_port_num;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));

        /* The port must have been stopped before. */
        ASSERT(!(port_cmd_status & AHCI_CMD_STATUS_ST));

        if (!(port_cmd_status & AHCI_CMD_STATUS_PMA)) {
                /* set PMA bit */
                ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
                    port_cmd_status|AHCI_CMD_STATUS_PMA);

                AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
                    "ahci_alloc_pmult: "
                    "PxCMD.PMA bit set at port %d.", port);
        }

        /* Allocate port multiplier information structure */
        if (ahci_portp->ahciport_pmult_info == NULL) {
                ahci_portp->ahciport_pmult_info = (ahci_pmult_info_t *)
                    kmem_zalloc(sizeof (ahci_pmult_info_t), KM_SLEEP);
        }

        ASSERT(ahci_portp->ahciport_pmult_info != NULL);
}

/*
 * ahci_dealloc_pmult()
 * 1. Clearing related registers when a port multiplier is detached.
 *    (Clear PxCMD.PMA while PxCMD.ST is '0')
 * 2. Deallocate ahci_pmult_info structure.
 *
 * NOTE: Must stop port before the function is called.
 */
static void
ahci_dealloc_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
{
        uint32_t port_cmd_status;
        uint8_t port = ahci_portp->ahciport_port_num;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));

        if (port_cmd_status & AHCI_CMD_STATUS_PMA) {
                /* Clear PMA bit */
                ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
                    (port_cmd_status & (~AHCI_CMD_STATUS_PMA)));

                AHCIDBG(AHCIDBG_INIT|AHCIDBG_PMULT, ahci_ctlp,
                    "ahci_dealloc_pmult: "
                    "PxCMD.PMA bit cleared at port %d.", port);
        }

        /* Release port multiplier information structure */
        if (ahci_portp->ahciport_pmult_info != NULL) {
                kmem_free(ahci_portp->ahciport_pmult_info,
                    sizeof (ahci_pmult_info_t));
                ahci_portp->ahciport_pmult_info = NULL;
        }
}

/*
 * Staggered Spin-up.
 */
static void
ahci_staggered_spin_up(ahci_ctl_t *ahci_ctlp, uint8_t port)
{
        uint32_t cap_status;
        uint32_t port_cmd_status;

        ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));

        cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));

        /* Check for staggered spin-up support */
        if (!(cap_status & AHCI_HBA_CAP_SSS))
                return;

        port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));

        /* If PxCMD.SUD == 1, no staggered spin-up is needed */
        if (port_cmd_status & AHCI_CMD_STATUS_SUD)
                return;

        AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "Spin-up at port %d", port);

        /* Set PxCMD.SUD */
        port_cmd_status |= AHCI_CMD_STATUS_SUD;
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
            port_cmd_status);
}

/*
 * The routine is to initialize a port. First put the port in NotRunning
 * state, then enable port interrupt and clear Serror register. And under
 * AHCI_ATTACH case, find device signature and then try to start the port.
 *
 * Called by
 *    1. ahci_initialize_controller
 *    2. ahci_intr_phyrdy_change (hotplug)
 */
static int
ahci_initialize_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    ahci_addr_t *addrp)
{
        uint32_t port_sstatus, port_task_file, port_cmd_status;
        uint8_t port = addrp->aa_port;
        boolean_t resuming = B_TRUE;    /*  processing DDI_RESUME */
        int ret;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        /* AHCI_ADDR_PORT: We've no idea of the attached device here.  */
        ASSERT(AHCI_ADDR_IS_PORT(addrp));

        /*
         * At the time being, only probe ports/devices and get the types of
         * attached devices during DDI_ATTACH. In fact, the device can be
         * changed during power state changes, but at the time being, we
         * don't support the situation.
         */
        if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
                resuming = B_FALSE;
        } else {
                /* check for DDI_RESUME case */
                mutex_exit(&ahci_portp->ahciport_mutex);
                mutex_enter(&ahci_ctlp->ahcictl_mutex);
                if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH)
                        resuming = B_FALSE;
                mutex_exit(&ahci_ctlp->ahcictl_mutex);
                mutex_enter(&ahci_portp->ahciport_mutex);
        }

        if (resuming) {
                /*
                 * During the resume, we need to set the PxCLB, PxCLBU, PxFB
                 * and PxFBU registers in case these registers were cleared
                 * during the suspend.
                 */
                AHCIDBG(AHCIDBG_PM, ahci_ctlp,
                    "ahci_initialize_port: port %d "
                    "set PxCLB, PxCLBU, PxFB and PxFBU "
                    "during resume", port);

                if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
                    AHCI_SUCCESS)
                        return (AHCI_FAILURE);
        }

        port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));

        AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_initialize_port: port %d ", port);

        /*
         * Check whether the port is in NotRunning state, if not,
         * put the port in NotRunning state
         */
        if (port_cmd_status &
            (AHCI_CMD_STATUS_ST |
            AHCI_CMD_STATUS_CR |
            AHCI_CMD_STATUS_FRE |
            AHCI_CMD_STATUS_FR)) {
                (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
                    ahci_portp, port);
        }

        /* Make sure the drive is spun-up */
        ahci_staggered_spin_up(ahci_ctlp, port);

        /* Disable interrupt */
        ahci_disable_port_intrs(ahci_ctlp, port);

        /* Device is unknown at first */
        AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);

        /* Disable the interface power management */
        ahci_disable_interface_pm(ahci_ctlp, port);

        port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));
        port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));

        /* Check physcial link status */
        if (SSTATUS_GET_IPM(port_sstatus) == SSTATUS_IPM_NODEV_NOPHYCOM ||
            SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_NOPHYCOM ||

            /* Check interface status */
            port_task_file & AHCI_TFD_STS_BSY ||
            port_task_file & AHCI_TFD_STS_DRQ ||

            /* Check whether port reset must be executed */
            ahci_ctlp->ahcictl_cap & AHCI_CAP_INIT_PORT_RESET ||

            /* Always reset port on RESUME */
            resuming != B_FALSE) {

                /* Something went wrong, we need do some reset things */
                ret = ahci_port_reset(ahci_ctlp, ahci_portp, addrp);

                /* Does port reset succeed on HBA port? */
                if (ret != AHCI_SUCCESS) {
                        AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_initialize_port:"
                            "port reset failed at port %d", port);
                        return (AHCI_FAILURE);
                }

                /* Is port failed? */
                if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
                    SATA_PSTATE_FAILED) {
                        AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_initialize_port: port %d state 0x%x",
                            port, ahci_portp->ahciport_port_state);
                        return (AHCI_FAILURE);
                }
        }

        AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
        AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "port %d is ready now.", port);

        /*
         * Try to get the device signature if the port is not empty.
         */
        if (!resuming && AHCIPORT_DEV_TYPE(ahci_portp, addrp) !=
            SATA_DTYPE_NONE)
                ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);

        /* Return directly if no device connected */
        if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_NONE) {
                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "No device connected to port %d", port);
                goto out;
        }

        /* If this is a port multiplier, we need do some initialization */
        if (AHCIPORT_DEV_TYPE(ahci_portp, addrp) == SATA_DTYPE_PMULT) {
                AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
                    "Port multiplier found at port %d", port);
                ahci_alloc_pmult(ahci_ctlp, ahci_portp);
        }

        /* Try to start the port */
        if (ahci_start_port(ahci_ctlp, ahci_portp, port)
            != AHCI_SUCCESS) {
                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "failed to start port %d", port);
                return (AHCI_FAILURE);
        }
out:
        /* Enable port interrupts */
        ahci_enable_port_intrs(ahci_ctlp, port);

        return (AHCI_SUCCESS);
}

/*
 *  Handle hardware defect, and check the capabilities. For example,
 *  power management capabilty and MSI capability.
 */
static int
ahci_config_space_init(ahci_ctl_t *ahci_ctlp)
{
        ushort_t caps_ptr, cap_count, cap;
#if AHCI_DEBUG
        ushort_t pmcap, pmcsr;
        ushort_t msimc;
#endif
        uint8_t revision;

        ahci_ctlp->ahcictl_venid =
            pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
            PCI_CONF_VENID);

        ahci_ctlp->ahcictl_devid =
            pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
            PCI_CONF_DEVID);

        /*
         * Modify dma_attr_align of ahcictl_buffer_dma_attr. For VT8251, those
         * controllers with 0x00 revision id work on 4-byte aligned buffer,
         * which is a bug and was fixed after 0x00 revision id controllers.
         *
         * Moreover, VT8251 cannot use multiple command slots in the command
         * list for non-queued commands because the previous register content
         * of PxCI can be re-written in the register write, so a flag will be
         * set to record this defect - AHCI_CAP_NO_MCMDLIST_NONQUEUE.
         *
         * For VT8251, software reset also has the same defect as the below
         * AMD/ATI chipset. That is, software reset will get failed if 0xf
         * is filled in pmport field. Therefore, another software reset need
         * to be done with 0 filled in pmport field.
         */
        if (ahci_ctlp->ahcictl_venid == VIA_VENID) {
                revision = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
                    PCI_CONF_REVID);
                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "revision id = 0x%x", revision);
                if (revision == 0x00) {
                        ahci_ctlp->ahcictl_buffer_dma_attr.dma_attr_align = 0x4;
                        AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                            "change ddi_attr_align to 0x4", NULL);
                }

                ahci_ctlp->ahcictl_cap |= AHCI_CAP_NO_MCMDLIST_NONQUEUE;
                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "VT8251 cannot use multiple command lists for "
                    "non-queued commands", NULL);

                ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;
        }

        /*
         * AMD/ATI SB600 (0x1002,0x4380) AHCI chipset doesn't support 64-bit
         * DMA addressing for communication memory descriptors though S64A bit
         * of CAP register declares it supports. Even though 64-bit DMA for
         * data buffer works on ASUS M2A-VM with newer BIOS, three other
         * motherboards are known not, so both AHCI_CAP_BUF_32BIT_DMA and
         * AHCI_CAP_COMMU_32BIT_DMA are set for this controller.
         *
         * Due to certain hardware issue, the chipset must do port reset during
         * initialization, otherwise, when retrieving device signature,
         * software reset will get time out. So AHCI_CAP_INIT_PORT_RESET flag
         * need to set.
         *
         * For this chipset software reset will get failure if the pmport of
         * Register FIS was set with SATA_PMULT_HOSTPORT (0xf) and no port
         * multiplier is connected to the port. In order to fix the issue,
         * AHCI_CAP_SRST_NO_HOSTPORT flag need to be set, and once software
         * reset got failure, the driver will try to do another software reset
         * with pmport 0.
         */
        if (ahci_ctlp->ahcictl_venid == 0x1002 &&
            ahci_ctlp->ahcictl_devid == 0x4380) {
                ahci_ctlp->ahcictl_cap |= AHCI_CAP_BUF_32BIT_DMA;
                ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
                ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
                ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;

                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "ATI SB600 cannot do 64-bit DMA for both data buffer and "
                    "communication memory descriptors though CAP indicates "
                    "support, so force it to use 32-bit DMA", NULL);
                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "ATI SB600 need to do a port reset during initialization",
                    NULL);
                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "ATI SB600 will get software reset failure if pmport "
                    "is set 0xf and no port multiplier is attached", NULL);
        }

        /*
         * AMD/ATI SB700/710/750/800 and SP5100 AHCI chipset share the same
         * vendor ID and device ID (0x1002,0x4391).
         *
         * SB700/750 AHCI chipset on some boards doesn't support 64-bit
         * DMA addressing for communication memory descriptors though S64A bit
         * of CAP register declares the support. However, it does support
         * 64-bit DMA for data buffer. So only AHCI_CAP_COMMU_32BIT_DMA is
         * set for this controller.
         *
         * SB710 has the same initialization issue as SB600, so it also need
         * a port reset. That is AHCI_CAP_INIT_PORT_RESET need to set for it.
         *
         * SB700 also has the same issue about software reset, and thus
         * AHCI_CAP_SRST_NO_HOSTPORT flag also is needed.
         */
        if (ahci_ctlp->ahcictl_venid == 0x1002 &&
            ahci_ctlp->ahcictl_devid == 0x4391) {
                ahci_ctlp->ahcictl_cap |= AHCI_CAP_COMMU_32BIT_DMA;
                ahci_ctlp->ahcictl_cap |= AHCI_CAP_INIT_PORT_RESET;
                ahci_ctlp->ahcictl_cap |= AHCI_CAP_SRST_NO_HOSTPORT;

                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "ATI SB700/750 cannot do 64-bit DMA for communication "
                    "memory descriptors though CAP indicates support, "
                    "so force it to use 32-bit DMA", NULL);
                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "ATI SB710 need to do a port reset during initialization",
                    NULL);
                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "ATI SB700 will get software reset failure if pmport "
                    "is set 0xf and no port multiplier is attached", NULL);
        }

        /*
         * Check if capabilities list is supported and if so,
         * get initial capabilities pointer and clear bits 0,1.
         */
        if (pci_config_get16(ahci_ctlp->ahcictl_pci_conf_handle,
            PCI_CONF_STAT) & PCI_STAT_CAP) {
                caps_ptr = P2ALIGN(pci_config_get8(
                    ahci_ctlp->ahcictl_pci_conf_handle,
                    PCI_CONF_CAP_PTR), 4);
        } else {
                caps_ptr = PCI_CAP_NEXT_PTR_NULL;
        }

        /*
         * Walk capabilities if supported.
         */
        for (cap_count = 0; caps_ptr != PCI_CAP_NEXT_PTR_NULL; ) {

                /*
                 * Check that we haven't exceeded the maximum number of
                 * capabilities and that the pointer is in a valid range.
                 */
                if (++cap_count > PCI_CAP_MAX_PTR) {
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "too many device capabilities", NULL);
                        return (AHCI_FAILURE);
                }
                if (caps_ptr < PCI_CAP_PTR_OFF) {
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "capabilities pointer 0x%x out of range",
                            caps_ptr);
                        return (AHCI_FAILURE);
                }

                /*
                 * Get next capability and check that it is valid.
                 * For now, we only support power management.
                 */
                cap = pci_config_get8(ahci_ctlp->ahcictl_pci_conf_handle,
                    caps_ptr);
                switch (cap) {
                case PCI_CAP_ID_PM:

                        /* power management supported */
                        ahci_ctlp->ahcictl_cap |= AHCI_CAP_PM;

                        /* Save PMCSR offset */
                        ahci_ctlp->ahcictl_pmcsr_offset = caps_ptr + PCI_PMCSR;

#if AHCI_DEBUG
                        pmcap = pci_config_get16(
                            ahci_ctlp->ahcictl_pci_conf_handle,
                            caps_ptr + PCI_PMCAP);
                        pmcsr = pci_config_get16(
                            ahci_ctlp->ahcictl_pci_conf_handle,
                            ahci_ctlp->ahcictl_pmcsr_offset);
                        AHCIDBG(AHCIDBG_PM, ahci_ctlp,
                            "Power Management capability found PCI_PMCAP "
                            "= 0x%x PCI_PMCSR = 0x%x", pmcap, pmcsr);
                        if ((pmcap & 0x3) == 0x3)
                                AHCIDBG(AHCIDBG_PM, ahci_ctlp,
                                    "PCI Power Management Interface "
                                    "spec 1.2 compliant", NULL);
#endif
                        break;

                case PCI_CAP_ID_MSI:
#if AHCI_DEBUG
                        msimc = pci_config_get16(
                            ahci_ctlp->ahcictl_pci_conf_handle,
                            caps_ptr + PCI_MSI_CTRL);
                        AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
                            "Message Signaled Interrupt capability found "
                            "MSICAP_MC.MMC = 0x%x", (msimc & 0xe) >> 1);
#endif
                        AHCIDBG(AHCIDBG_MSI, ahci_ctlp,
                            "MSI capability found", NULL);
                        break;

                case PCI_CAP_ID_PCIX:
                        AHCIDBG(AHCIDBG_PM, ahci_ctlp,
                            "PCI-X capability found", NULL);
                        break;

                case PCI_CAP_ID_PCI_E:
                        AHCIDBG(AHCIDBG_PM, ahci_ctlp,
                            "PCI Express capability found", NULL);
                        break;

                case PCI_CAP_ID_MSI_X:
                        AHCIDBG(AHCIDBG_PM, ahci_ctlp,
                            "MSI-X capability found", NULL);
                        break;

                case PCI_CAP_ID_SATA:
                        AHCIDBG(AHCIDBG_PM, ahci_ctlp,
                            "SATA capability found", NULL);
                        break;

                case PCI_CAP_ID_VS:
                        AHCIDBG(AHCIDBG_PM, ahci_ctlp,
                            "Vendor Specific capability found", NULL);
                        break;

                default:
                        AHCIDBG(AHCIDBG_PM, ahci_ctlp,
                            "unrecognized capability 0x%x", cap);
                        break;
                }

                /*
                 * Get next capabilities pointer and clear bits 0,1.
                 */
                caps_ptr = P2ALIGN(pci_config_get8(
                    ahci_ctlp->ahcictl_pci_conf_handle,
                    (caps_ptr + PCI_CAP_NEXT_PTR)), 4);
        }

        return (AHCI_SUCCESS);
}

/*
 * Read/Write a register at port multiplier by SATA READ PORTMULT / SATA WRITE
 * PORTMULT command. SYNC & POLLING mode is used.
 */
static int
ahci_rdwr_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
    uint8_t regn, uint32_t *pregv, uint8_t type)
{
        ahci_port_t *ahci_portp;
        ahci_addr_t pmult_addr;
        sata_pkt_t *spkt;
        sata_cmd_t *scmd;
        sata_device_t sata_device;
        uint8_t port = addrp->aa_port;
        uint8_t pmport = addrp->aa_pmport;
        uint8_t cport;
        uint32_t intr_mask;
        int rval;
        char portstr[10];

        SET_PORTSTR(portstr, addrp);
        cport = ahci_ctlp->ahcictl_port_to_cport[port];
        ahci_portp = ahci_ctlp->ahcictl_ports[port];

        ASSERT(AHCI_ADDR_IS_PMPORT(addrp) || AHCI_ADDR_IS_PMULT(addrp));
        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        /* Check the existence of the port multiplier */
        if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT)
                return (AHCI_FAILURE);

        /* Request a READ/WRITE PORTMULT sata packet. */
        bzero(&sata_device, sizeof (sata_device_t));
        sata_device.satadev_addr.cport = cport;
        sata_device.satadev_addr.pmport = pmport;
        sata_device.satadev_addr.qual = SATA_ADDR_PMULT;
        sata_device.satadev_rev = SATA_DEVICE_REV;

        /*
         * Make sure no command is outstanding here. All R/W PMULT requests
         * come from
         *
         * 1. ahci_attach()
         *    The port should be empty.
         *
         * 2. ahci_tran_probe_port()
         *    Any request from SATA framework (via ahci_tran_start) should be
         *    rejected if R/W PMULT command is outstanding.
         *
         *    If we are doing mopping, do not check those flags because no
         *    command will be actually outstanding.
         *
         *    If the port has been occupied by any other commands, the probe
         *    function will return a SATA_RETRY. SATA framework will retry
         *    later.
         */
        if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
                AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
                    "R/W PMULT failed: R/W PMULT in progress at port %d.",
                    port, ahci_portp->ahciport_flags);
                return (AHCI_FAILURE);
        }

        if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) && (
            ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
            NCQ_CMD_IN_PROGRESS(ahci_portp) ||
            NON_NCQ_CMD_IN_PROGRESS(ahci_portp))) {
                AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
                    "R/W PMULT failed: port %d is occupied (flags 0x%x).",
                    port, ahci_portp->ahciport_flags);
                return (AHCI_FAILURE);
        }

        /*
         * The port multiplier is gone. This may happen when
         * 1. Cutting off the power of an enclosure. The device lose the power
         *    before port multiplier.
         * 2. Disconnecting the port multiplier during hot-plugging a sub-drive.
         *
         * The issued command should be aborted and the following command
         * should not be continued.
         */
        if (!(ahci_portp->ahciport_port_state & SATA_STATE_READY)) {
                AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
                    "READ/WRITE PMULT failed: "
                    "port-mult is removed from port %d", port);
                return (AHCI_FAILURE);
        }

        ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDWR_PMULT;

        spkt = sata_get_rdwr_pmult_pkt(ahci_ctlp->ahcictl_dip,
            &sata_device, regn, *pregv, type);

        /*
         * READ/WRITE PORTMULT command is intended to sent to the control port
         * of the port multiplier.
         */
        AHCI_ADDR_SET_PMULT(&pmult_addr, addrp->aa_port);

        ahci_portp->ahciport_rdwr_pmult_pkt = spkt;

        /* No interrupt here. Store the interrupt enable mask. */
        intr_mask = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);

        rval = ahci_do_sync_start(ahci_ctlp, ahci_portp, &pmult_addr, spkt);

        if (rval == AHCI_SUCCESS &&
            spkt->satapkt_reason == SATA_PKT_COMPLETED) {
                if (type == SATA_RDWR_PMULT_PKT_TYPE_READ) {
                        scmd = &spkt->satapkt_cmd;
                        *pregv = scmd->satacmd_lba_high_lsb << 24 |
                            scmd->satacmd_lba_mid_lsb << 16 |
                            scmd->satacmd_lba_low_lsb << 8 |
                            scmd->satacmd_sec_count_lsb;
                }
        } else {
                /* Failed or not completed. */
                AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
                    "ahci_rdwr_pmult: cannot [%s] %s[%d] at port %s",
                    type == SATA_RDWR_PMULT_PKT_TYPE_READ?"Read":"Write",
                    AHCI_ADDR_IS_PMULT(addrp)?"gscr":"pscr", regn, portstr);
                rval = AHCI_FAILURE;
        }

        /* Restore the interrupt mask */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), intr_mask);

        ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDWR_PMULT;
        ahci_portp->ahciport_rdwr_pmult_pkt = NULL;
        sata_free_rdwr_pmult_pkt(spkt);
        return (rval);
}

static int
ahci_read_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
    uint8_t regn, uint32_t *pregv)
{
        return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, pregv,
            SATA_RDWR_PMULT_PKT_TYPE_READ);
}

static int
ahci_write_pmult(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
    uint8_t regn, uint32_t regv)
{
        return ahci_rdwr_pmult(ahci_ctlp, addrp, regn, &regv,
            SATA_RDWR_PMULT_PKT_TYPE_WRITE);
}

#define READ_PMULT(addrp, r, pv, out)                                   \
        if (ahci_read_pmult(ahci_ctlp, addrp, r, pv) != AHCI_SUCCESS)   \
                goto out;

#define WRITE_PMULT(addrp, r, v, out)                                   \
        if (ahci_write_pmult(ahci_ctlp, addrp, r, v) != AHCI_SUCCESS)   \
                goto out;

/*
 * Update sata registers on port multiplier, including GSCR/PSCR registers.
 * ahci_update_pmult_gscr()
 * ahci_update_pmult_pscr()
 */
static int
ahci_update_pmult_gscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
    sata_pmult_gscr_t *sg)
{
        ASSERT(MUTEX_HELD(
            &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));

        READ_PMULT(addrp, SATA_PMULT_GSCR0, &sg->gscr0, err);
        READ_PMULT(addrp, SATA_PMULT_GSCR1, &sg->gscr1, err);
        READ_PMULT(addrp, SATA_PMULT_GSCR2, &sg->gscr2, err);
        READ_PMULT(addrp, SATA_PMULT_GSCR64, &sg->gscr64, err);

        return (AHCI_SUCCESS);

err:    /* R/W PMULT error */
        return (AHCI_FAILURE);
}

static int
ahci_update_pmult_pscr(ahci_ctl_t *ahci_ctlp, ahci_addr_t *addrp,
    sata_device_t *sd)
{
        ASSERT(AHCI_ADDR_IS_PMPORT(addrp));
        ASSERT(MUTEX_HELD(
            &ahci_ctlp->ahcictl_ports[addrp->aa_port]->ahciport_mutex));

        READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &sd->satadev_scr.sstatus, err);
        READ_PMULT(addrp, SATA_PMULT_REG_SERR, &sd->satadev_scr.serror, err);
        READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &sd->satadev_scr.scontrol, err);
        READ_PMULT(addrp, SATA_PMULT_REG_SACT, &sd->satadev_scr.sactive, err);

        return (AHCI_SUCCESS);

err:    /* R/W PMULT error */
        return (AHCI_FAILURE);
}

/*
 * ahci_initialize_pmult()
 *
 * Initialize a port multiplier, including
 * 1. Enable FEATURES register at port multiplier. (SATA Chp.16)
 * 2. Redefine MASK register. (SATA Chap 16.?)
 */
static int
ahci_initialize_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    ahci_addr_t *addrp, sata_device_t *sd)
{
        sata_pmult_gscr_t sg;
        uint32_t gscr64;
        uint8_t port = addrp->aa_port;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
            "[Initialize] Port-multiplier at port %d.", port);

        /*
         * Enable features of port multiplier. Currently only
         * Asynchronous Notification is enabled.
         */
        /* Check gscr64 for supported features. */
        READ_PMULT(addrp, SATA_PMULT_GSCR64, &gscr64, err);

        if (gscr64 & SATA_PMULT_CAP_SNOTIF) {
                AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
                    "port %d: Port Multiplier supports "
                    "Asynchronous Notification.", port);

                /* Write to gscr96 to enabled features */
                WRITE_PMULT(addrp, SATA_PMULT_GSCR96,
                    SATA_PMULT_CAP_SNOTIF, err);

                ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
                    AHCI_SNOTIF_CLEAR_ALL);
                AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
                    "port %d: PMult PxSNTF cleared.", port);

        }

        /*
         * Now we need to update gscr33 register to enable hot-plug interrupt
         * for sub devices behind port multiplier.
         */
        WRITE_PMULT(addrp, SATA_PMULT_GSCR33, (0x1ffff), err);
        AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
            "port %d: gscr33 mask set to %x.", port, (0x1ffff));

        /*
         * Fetch the number of device ports of the port multiplier
         */
        if (ahci_update_pmult_gscr(ahci_ctlp, addrp, &sg) != AHCI_SUCCESS)
                return (AHCI_FAILURE);

        /* Register the port multiplier to SATA Framework. */
        mutex_exit(&ahci_portp->ahciport_mutex);
        sata_register_pmult(ahci_ctlp->ahcictl_dip, sd, &sg);
        mutex_enter(&ahci_portp->ahciport_mutex);

        ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports =
            sd->satadev_add_info & SATA_PMULT_PORTNUM_MASK;

        AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
            "port %d: pmult sub-port number updated to %x.", port,
            ahci_portp->ahciport_pmult_info->ahcipmi_num_dev_ports);

        /* Till now port-mult is successfully initialized */
        ahci_portp->ahciport_port_state |= SATA_DSTATE_PMULT_INIT;
        return (AHCI_SUCCESS);

err:    /* R/W PMULT error */
        return (AHCI_FAILURE);
}

/*
 * Initialize a port multiplier port. According to spec, firstly we need
 * issue a COMRESET, then a software reset to get its signature.
 *
 * NOTE: This function should only be called in ahci_probe_pmport()
 */
static int
ahci_initialize_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    ahci_addr_t *addrp)
{
        uint32_t finished_tags = 0, reset_tags = 0, slot_status = 0;
        uint8_t port = addrp->aa_port;
        uint8_t pmport = addrp->aa_pmport;
        int ret = AHCI_FAILURE;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
        ASSERT(AHCI_ADDR_IS_PMPORT(addrp));

        AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_initialize_pmport: port %d:%d", port, pmport);

        /* Check HBA port state */
        if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
                AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_initialize_pmport:"
                    "port %d:%d Port Multiplier is failed.",
                    port, pmport);
                return (AHCI_FAILURE);
        }

        if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_HOTPLUG) {
                return (AHCI_FAILURE);
        }
        ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;

        /* Checking for outstanding commands */
        if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
                reset_tags = slot_status & AHCI_SLOT_MASK(ahci_ctlp);
        } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
                reset_tags = slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
        }

        ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
        ahci_portp->ahciport_mop_in_progress++;

        /* Clear status */
        AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_UNKNOWN);

        /* Firstly assume an unknown device */
        AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);

        ahci_disable_port_intrs(ahci_ctlp, port);

        /* port reset is necessary for port multiplier port */
        if (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp) != AHCI_SUCCESS) {
                AHCIDBG(AHCIDBG_INIT|AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_initialize_pmport:"
                    "port reset failed at port %d:%d",
                    port, pmport);
                goto out;
        }

        /* Is port failed? */
        if (AHCIPORT_GET_STATE(ahci_portp, addrp) &
            SATA_PSTATE_FAILED) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_initialize_pmport: port %d:%d failed. "
                    "state = 0x%x", port, pmport,
                    ahci_portp->ahciport_port_state);
                goto out;
        }

        /* Is there any device attached? */
        if (AHCIPORT_GET_DEV_TYPE(ahci_portp, addrp)
            == SATA_DTYPE_NONE) {
                /* Do not waste time on an empty port */
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                    "ahci_initialize_pmport: No device is found "
                    "at port %d:%d", port, pmport);
                ret = AHCI_SUCCESS;
                goto out;
        }

        AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_STATE_READY);
        AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
            "port %d:%d is ready now.", port, pmport);

        /*
         * Till now we can assure a device attached to that HBA port and work
         * correctly. Now try to get the device signature. This is an optional
         * step. If failed, unknown device is assumed, then SATA module will
         * continue to use IDENTIFY DEVICE to get the information of the
         * device.
         */
        ahci_find_dev_signature(ahci_ctlp, ahci_portp, addrp);

        ret = AHCI_SUCCESS;

out:
        /* Next try to mop the pending commands */
        if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp))
                finished_tags = ahci_portp->ahciport_pending_tags &
                    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
        else if (NCQ_CMD_IN_PROGRESS(ahci_portp))
                finished_tags = ahci_portp->ahciport_pending_ncq_tags &
                    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
        reset_tags &= ~finished_tags;

        ahci_mop_commands(ahci_ctlp,
            ahci_portp,
            slot_status,
            0, /* failed tags */
            0, /* timeout tags */
            0, /* aborted tags */
            reset_tags); /* reset tags */

        /* Clear PxSNTF register if supported. */
        if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
                ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
                    AHCI_SNOTIF_CLEAR_ALL);
        }

        ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;
        ahci_enable_port_intrs(ahci_ctlp, port);
        return (ret);
}

/*
 * ahci_probe_pmult()
 *
 * This function will be called to probe a port multiplier, which will
 * handle hotplug events on port multiplier ports.
 *
 * NOTE: Only called from ahci_tran_probe_port()
 */
static int
ahci_probe_pmult(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    ahci_addr_t *addrp)
{
        sata_device_t sdevice;
        ahci_addr_t pmport_addr;
        uint32_t gscr32, port_hotplug_tags;
        uint32_t pmport_sstatus;
        int dev_exists_now = 0, dev_existed_previously = 0;
        uint8_t port = addrp->aa_port;
        int npmport;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        /* The bits in GSCR32 refers to the pmport that has a hot-plug event. */
        READ_PMULT(addrp, SATA_PMULT_GSCR32, &gscr32, err);
        port_hotplug_tags = gscr32 & AHCI_PMPORT_MASK(ahci_portp);

        do {
                npmport = ddi_ffs(port_hotplug_tags) - 1;
                if (npmport == -1)
                        /* no pending hot plug events. */
                        return (AHCI_SUCCESS);

                AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
                    "hot-plug event at port %d:%d", port, npmport);

                AHCI_ADDR_SET_PMPORT(&pmport_addr, port, (uint8_t)npmport);

                /* Check previous device at that port */
                if (AHCIPORT_GET_DEV_TYPE(ahci_portp, &pmport_addr)
                    != SATA_DTYPE_NONE)
                        dev_existed_previously = 1;

                /* PxSStatus tells the presence of device. */
                READ_PMULT(&pmport_addr, SATA_PMULT_REG_SSTS,
                    &pmport_sstatus, err);

                if (SSTATUS_GET_DET(pmport_sstatus) ==
                    SSTATUS_DET_DEVPRE_PHYCOM)
                        dev_exists_now = 1;

                /*
                 * Clear PxSERR is critical. The transition from 0 to 1 will
                 * emit a FIS which generates an asynchronous notification
                 * event at controller. If we fail to clear the PxSERR, the
                 * Async Notif events will no longer be activated on this
                 * pmport.
                 */
                WRITE_PMULT(&pmport_addr, SATA_PMULT_REG_SERR,
                    AHCI_SERROR_CLEAR_ALL, err);

                bzero((void *)&sdevice, sizeof (sata_device_t));
                sdevice.satadev_addr.cport = ahci_ctlp->
                    ahcictl_port_to_cport[port];
                sdevice.satadev_addr.qual = SATA_ADDR_PMPORT;
                sdevice.satadev_addr.pmport = (uint8_t)npmport;
                sdevice.satadev_state = SATA_PSTATE_PWRON;

                AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
                    "[Existence] %d -> %d", dev_existed_previously,
                    dev_exists_now);

                if (dev_exists_now) {
                        if (dev_existed_previously) {
                                /* Link (may) not change: Exist -> Exist * */
                                AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
                                    "ahci_probe_pmult: port %d:%d "
                                    "device link lost/established",
                                    port, npmport);

                                mutex_exit(&ahci_portp->ahciport_mutex);
                                sata_hba_event_notify(
                                    ahci_ctlp->ahcictl_sata_hba_tran->
                                    sata_tran_hba_dip,
                                    &sdevice,
                                    SATA_EVNT_LINK_LOST|
                                    SATA_EVNT_LINK_ESTABLISHED);
                                mutex_enter(&ahci_portp->ahciport_mutex);
                        } else {
                                /* Link change: None -> Exist */
                                AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
                                    "ahci_probe_pmult: port %d:%d "
                                    "device link established", port, npmport);

                                /* Clear port state */
                                AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
                                    SATA_STATE_UNKNOWN);
                                AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
                                    "ahci_probe_pmult: port %d "
                                    "ahciport_port_state [Cleared].", port);

                                mutex_exit(&ahci_portp->ahciport_mutex);
                                sata_hba_event_notify(
                                    ahci_ctlp->ahcictl_sata_hba_tran->
                                    sata_tran_hba_dip,
                                    &sdevice,
                                    SATA_EVNT_LINK_ESTABLISHED);
                                mutex_enter(&ahci_portp->ahciport_mutex);
                        }
                } else { /* No device exists now */
                        if (dev_existed_previously) {

                                /* Link change: Exist -> None */
                                AHCIDBG(AHCIDBG_EVENT|AHCIDBG_PMULT, ahci_ctlp,
                                    "ahci_probe_pmult: port %d:%d "
                                    "device link lost", port, npmport);

                                /* An existing device is lost. */
                                AHCIPORT_SET_STATE(ahci_portp, &pmport_addr,
                                    SATA_STATE_UNKNOWN);
                                AHCIPORT_SET_DEV_TYPE(ahci_portp, &pmport_addr,
                                    SATA_DTYPE_NONE);

                                mutex_exit(&ahci_portp->ahciport_mutex);
                                sata_hba_event_notify(
                                    ahci_ctlp->ahcictl_sata_hba_tran->
                                    sata_tran_hba_dip,
                                    &sdevice,
                                    SATA_EVNT_LINK_LOST);
                                mutex_enter(&ahci_portp->ahciport_mutex);
                        }
                }

                CLEAR_BIT(port_hotplug_tags, npmport);
        } while (port_hotplug_tags != 0);

        return (AHCI_SUCCESS);

err:    /* R/W PMULT error */
        return (AHCI_FAILURE);
}

/*
 * Probe and initialize a port multiplier port.
 * A port multiplier port could only be initilaizer here.
 */
static int
ahci_probe_pmport(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    ahci_addr_t *addrp, sata_device_t *sd)
{
        uint32_t port_state;
        uint8_t port = addrp->aa_port;
        ahci_addr_t addr_pmult;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        /*
         * Check the parent - port multiplier first.
         */

        /*
         * Parent port multiplier might have been removed. This event will be
         * ignored and failure.
         */
        if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
            ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
                AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
                    "ahci_tran_probe_port: "
                    "parent device removed, ignore event.", NULL);

                return (AHCI_FAILURE);
        }

        /* The port is ready? */
        port_state = ahci_portp->ahciport_port_state;
        if (!(port_state & SATA_STATE_READY)) {
                AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
                    "ahci_tran_probe_port: "
                    "parent port-mult is NOT ready.", NULL);

                if (ahci_restart_port_wait_till_ready(ahci_ctlp,
                    ahci_portp, port, AHCI_PORT_RESET, NULL) !=
                    AHCI_SUCCESS) {
                        AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
                            "ahci_tran_probe_port: "
                            "restart port-mult failed.", NULL);
                        return (AHCI_FAILURE);
                }
        }

        /*
         * If port-mult is restarted due to some reason, we need
         * re-initialized the PMult.
         */
        if (!(port_state & SATA_DSTATE_PMULT_INIT)) {
                /* Initialize registers on a port multiplier */
                AHCI_ADDR_SET_PMULT(&addr_pmult, addrp->aa_port);
                if (ahci_initialize_pmult(ahci_ctlp, ahci_portp,
                    &addr_pmult, sd) != AHCI_SUCCESS)
                        return (AHCI_FAILURE);
        }

        /*
         * Then we check the port-mult port
         */
        /* Is this pmport initialized? */
        port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
        if (!(port_state & SATA_STATE_READY)) {

                /* ahci_initialize_pmport() will set READY state */
                if (ahci_initialize_pmport(ahci_ctlp,
                    ahci_portp, addrp) != AHCI_SUCCESS)
                        return (AHCI_FAILURE);
        }

        return (AHCI_SUCCESS);
}

/*
 * AHCI device reset ...; a single device on one of the ports is reset,
 * but the HBA and physical communication remain intact. This is the
 * least intrusive.
 *
 * When issuing a software reset sequence, there should not be other
 * commands in the command list, so we will first clear and then re-set
 * PxCMD.ST to clear PxCI. And before issuing the software reset,
 * the port must be idle and PxTFD.STS.BSY and PxTFD.STS.DRQ must be
 * cleared unless command list override (PxCMD.CLO) is supported.
 */
static int
ahci_software_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    ahci_addr_t *addrp)
{
        ahci_fis_h2d_register_t *h2d_register_fisp;
        ahci_cmd_table_t *cmd_table;
        ahci_cmd_header_t *cmd_header;
        uint32_t port_cmd_status, port_cmd_issue, port_task_file;
        int slot, loop_count;
        uint8_t port = addrp->aa_port;
        uint8_t pmport = addrp->aa_pmport;
        int rval = AHCI_FAILURE;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "port %d:%d device software resetting (FIS)", port, pmport);

        /* First clear PxCMD.ST (AHCI v1.2 10.4.1) */
        if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
            port) != AHCI_SUCCESS) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_software_reset: cannot stop HBA port %d.", port);
                goto out;
        }

        /* Check PxTFD.STS.BSY and PxTFD.STS.DRQ */
        port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));

        if (port_task_file & AHCI_TFD_STS_BSY ||
            port_task_file & AHCI_TFD_STS_DRQ) {
                if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_SCLO)) {
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "PxTFD.STS.BSY/DRQ is set (PxTFD=0x%x), "
                            "cannot issue a software reset.", port_task_file);
                        goto out;
                }

                /*
                 * If HBA Support CLO, as Command List Override (CAP.SCLO is
                 * set), PxCMD.CLO bit should be set before set PxCMD.ST, in
                 * order to clear PxTFD.STS.BSY and PxTFD.STS.DRQ.
                 */
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "PxTFD.STS.BSY/DRQ is set, try SCLO.", NULL)

                port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
                ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
                    port_cmd_status|AHCI_CMD_STATUS_CLO);

                /* Waiting till PxCMD.SCLO bit is cleared */
                loop_count = 0;
                do {
                        /* Wait for 10 millisec */
                        drv_usecwait(AHCI_10MS_USECS);

                        /* We are effectively timing out after 1 sec. */
                        if (loop_count++ > 100) {
                                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                                    "SCLO time out. port %d is busy.", port);
                                goto out;
                        }

                        port_cmd_status =
                            ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
                } while (port_cmd_status & AHCI_CMD_STATUS_CLO);

                /* Re-check */
                port_task_file = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
                if (port_task_file & AHCI_TFD_STS_BSY ||
                    port_task_file & AHCI_TFD_STS_DRQ) {
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "SCLO cannot clear PxTFD.STS.BSY/DRQ (PxTFD=0x%x)",
                            port_task_file);
                        goto out;
                }
        }

        /* Then start port */
        if (ahci_start_port(ahci_ctlp, ahci_portp, port)
            != AHCI_SUCCESS) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_software_reset: cannot start AHCI port %d.", port);
                goto out;
        }

        /*
         * When ahci_port.ahciport_mop_in_progress is set, A non-zero
         * ahci_port.ahciport_pending_ncq_tags may fail
         * ahci_claim_free_slot(). Actually according to spec, by clearing
         * PxCMD.ST there is no command outstanding while executing software
         * reseting. Hence we directly use slot 0 instead of
         * ahci_claim_free_slot().
         */
        slot = 0;

        /* Now send the first H2D Register FIS with SRST set to 1 */
        cmd_table = ahci_portp->ahciport_cmd_tables[slot];
        bzero((void *)cmd_table, ahci_cmd_table_size);

        h2d_register_fisp =
            &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);

        SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
        SET_FIS_PMP(h2d_register_fisp, pmport);
        SET_FIS_DEVCTL(h2d_register_fisp, SATA_DEVCTL_SRST);

        /* Set Command Header in Command List */
        cmd_header = &ahci_portp->ahciport_cmd_list[slot];
        BZERO_DESCR_INFO(cmd_header);
        BZERO_PRD_BYTE_COUNT(cmd_header);
        SET_COMMAND_FIS_LENGTH(cmd_header, 5);
        SET_PORT_MULTI_PORT(cmd_header, pmport);

        SET_CLEAR_BUSY_UPON_R_OK(cmd_header, 1);
        SET_RESET(cmd_header, 1);
        SET_WRITE(cmd_header, 1);

        (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
            0,
            ahci_cmd_table_size,
            DDI_DMA_SYNC_FORDEV);

        (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
            slot * sizeof (ahci_cmd_header_t),
            sizeof (ahci_cmd_header_t),
            DDI_DMA_SYNC_FORDEV);

        /* Indicate to the HBA that a command is active. */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
            (0x1 << slot));

        loop_count = 0;

        /* Loop till the first command is finished */
        do {
                port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));

                /* We are effectively timing out after 1 sec. */
                if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "the first SRST FIS is timed out, "
                            "loop_count = %d", loop_count);
                        goto out;
                }
                /* Wait for 10 millisec */
                drv_usecwait(AHCI_10MS_USECS);
        } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));

        AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
            "ahci_software_reset: 1st loop count: %d, "
            "port_cmd_issue = 0x%x, slot = 0x%x",
            loop_count, port_cmd_issue, slot);

        /* According to ATA spec, we need wait at least 5 microsecs here. */
        drv_usecwait(AHCI_1MS_USECS);

        /* Now send the second H2D Register FIS with SRST cleard to zero */
        cmd_table = ahci_portp->ahciport_cmd_tables[slot];
        bzero((void *)cmd_table, ahci_cmd_table_size);

        h2d_register_fisp =
            &(cmd_table->ahcict_command_fis.ahcifc_fis.ahcifc_h2d_register);

        SET_FIS_TYPE(h2d_register_fisp, AHCI_H2D_REGISTER_FIS_TYPE);
        SET_FIS_PMP(h2d_register_fisp, pmport);

        /* Set Command Header in Command List */
        cmd_header = &ahci_portp->ahciport_cmd_list[slot];
        BZERO_DESCR_INFO(cmd_header);
        BZERO_PRD_BYTE_COUNT(cmd_header);
        SET_COMMAND_FIS_LENGTH(cmd_header, 5);
        SET_PORT_MULTI_PORT(cmd_header, pmport);

        SET_WRITE(cmd_header, 1);

        (void) ddi_dma_sync(ahci_portp->ahciport_cmd_tables_dma_handle[slot],
            0,
            ahci_cmd_table_size,
            DDI_DMA_SYNC_FORDEV);

        (void) ddi_dma_sync(ahci_portp->ahciport_cmd_list_dma_handle,
            slot * sizeof (ahci_cmd_header_t),
            sizeof (ahci_cmd_header_t),
            DDI_DMA_SYNC_FORDEV);

        /* Indicate to the HBA that a command is active. */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port),
            (0x1 << slot));

        loop_count = 0;

        /* Loop till the second command is finished */
        do {
                port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));

                /* We are effectively timing out after 1 sec. */
                if (loop_count++ > AHCI_POLLRATE_PORT_SOFTRESET) {
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "the second SRST FIS is timed out, "
                            "loop_count = %d", loop_count);
                        goto out;
                }

                /* Wait for 10 millisec */
                drv_usecwait(AHCI_10MS_USECS);
        } while (port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp) & (0x1 << slot));

        AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
            "ahci_software_reset: 2nd loop count: %d, "
            "port_cmd_issue = 0x%x, slot = 0x%x",
            loop_count, port_cmd_issue, slot);

        if ((ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) ||
            (ahci_check_port_handle(ahci_ctlp, port) != DDI_SUCCESS)) {
                ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
                    DDI_SERVICE_UNAFFECTED);
                goto out;
        }

        rval = AHCI_SUCCESS;
out:
        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
            "ahci_software_reset: %s at port %d:%d",
            rval == AHCI_SUCCESS ? "succeed" : "failed",
            port, pmport);

        return (rval);
}

/*
 * AHCI port reset ...; the physical communication between the HBA and device
 * on a port are disabled. This is more intrusive.
 *
 * When an HBA or port reset occurs, Phy communication is going to
 * be re-established with the device through a COMRESET followed by the
 * normal out-of-band communication sequence defined in Serial ATA. At
 * the end of reset, the device, if working properly, will send a D2H
 * Register FIS, which contains the device signature. When the HBA receives
 * this FIS, it updates PxTFD.STS and PxTFD.ERR register fields, and updates
 * the PxSIG register with the signature.
 *
 * NOTE: It is expected both PxCMD.ST and PxCMD.CR are cleared before the
 * function is called. If not, it is assumed the interface is in hung
 * condition.
 */
static int
ahci_port_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    ahci_addr_t *addrp)
{
        ahci_addr_t pmult_addr;
        uint32_t port_cmd_status;
        uint32_t port_scontrol, port_sstatus;
        uint32_t port_task_file;
        uint32_t port_state;
        uint8_t port = addrp->aa_port;

        int loop_count;
        int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        /* Target is a port multiplier port? */
        if (AHCI_ADDR_IS_PMPORT(addrp))
                return (ahci_pmport_reset(ahci_ctlp, ahci_portp, addrp));

        /* Otherwise it must be an HBA port. */
        ASSERT(AHCI_ADDR_IS_PORT(addrp));

        AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
            "Port %d port resetting...", port);
        ahci_portp->ahciport_port_state = 0;

        port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));

        /*
         * According to the spec, SUD bit should be set here,
         * but JMicron JMB363 doesn't follow it, so print
         * a debug message.
         */
        if (!(port_cmd_status & AHCI_CMD_STATUS_SUD))
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_port_reset: port %d SUD bit not set", port);

        port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
        SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);

        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
            port_scontrol);

        /* Enable PxCMD.FRE to read device */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
            port_cmd_status|AHCI_CMD_STATUS_FRE);

        /*
         * The port enters P:StartComm state, and the HBA tells the link layer
         * to start communication, which involves sending COMRESET to the
         * device. And the HBA resets PxTFD.STS to 7Fh.
         *
         * Give time for COMRESET to percolate, according to the AHCI
         * spec, software shall wait at least 1 millisecond before
         * clearing PxSCTL.DET
         */
        drv_usecwait(AHCI_1MS_USECS * 2);

        /* Fetch the SCONTROL again and rewrite the DET part with 0 */
        port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
        SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port),
            port_scontrol);

        /*
         * When a COMINIT is received from the device, then the port enters
         * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
         * PxSSTS.DET to 1h to indicate a device is detected but communication
         * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
         * a COMINIT has been received.
         */
        /*
         * The DET field is valid only if IPM field indicates
         * that the interface is in active state.
         */
        loop_count = 0;
        for (;;) {
                port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));

                if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
                        /*
                         * If the interface is not active, the DET field
                         * is considered not accurate. So we want to
                         * continue looping.
                         */
                        SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
                }

                if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM)
                        break;

                if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
                        /*
                         * We are effectively timing out after 0.1 sec.
                         */
                        break;
                }

                /* Wait for 10 millisec */
                drv_usecwait(AHCI_10MS_USECS);
        }

        AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
            "ahci_port_reset: 1st loop count: %d, "
            "port_sstatus = 0x%x port %d",
            loop_count, port_sstatus, port);

        if (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM) {
                /*
                 * Either the port is not active or there
                 * is no device present.
                 */
                AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
                return (AHCI_SUCCESS);
        }

        /* Clear port serror register for the port */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
            AHCI_SERROR_CLEAR_ALL);

        /*
         * Devices should return a FIS contains its signature to HBA after
         * COMINIT signal. Check whether a D2H Register FIS is received by
         * polling PxTFD.STS.
         */
        loop_count = 0;
        for (;;) {
                port_task_file =
                    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));

                if ((port_task_file & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ |
                    AHCI_TFD_STS_ERR)) == 0)
                        break;

                if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
                        /*
                         * We are effectively timing out after 11 sec.
                         */
                        cmn_err(CE_WARN, "!ahci%d: ahci_port_reset port %d "
                            "the device hardware has been initialized and "
                            "the power-up diagnostics failed",
                            instance, port);

                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_port_reset: "
                            "port %d: some or all of BSY, DRQ and ERR in "
                            "PxTFD.STS are not clear. We need another "
                            "software reset.", port);

                        /* Clear port serror register for the port */
                        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
                            (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
                            AHCI_SERROR_CLEAR_ALL);

                        AHCI_ADDR_SET_PMULT(&pmult_addr, port);

                        /* Try another software reset. */
                        if (ahci_software_reset(ahci_ctlp, ahci_portp,
                            &pmult_addr) != AHCI_SUCCESS) {
                                AHCIPORT_SET_STATE(ahci_portp, addrp,
                                    SATA_PSTATE_FAILED);
                                return (AHCI_FAILURE);
                        }
                        break;
                }

                /* Wait for 10 millisec */
                drv_usecwait(AHCI_10MS_USECS);
        }

        AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
            "ahci_port_reset: 2nd loop count: %d, "
            "port_task_file = 0x%x port %d",
            loop_count, port_task_file, port);

        /* Clear port serror register for the port */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
            AHCI_SERROR_CLEAR_ALL);

        /* Set port as ready */
        port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
        AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);

        AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
            "ahci_port_reset: succeed at port %d.", port);
        return (AHCI_SUCCESS);
}

/*
 * COMRESET on a port multiplier port.
 *
 * NOTE: Only called in ahci_port_reset()
 */
static int
ahci_pmport_reset(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    ahci_addr_t *addrp)
{
        uint32_t port_scontrol, port_sstatus, port_serror;
        uint32_t port_cmd_status, port_intr_status;
        uint32_t port_state;
        uint8_t port = addrp->aa_port;
        uint8_t pmport = addrp->aa_pmport;
        int loop_count;
        int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);

        AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
            "port %d:%d: pmport resetting", port, pmport);

        /* Initialize pmport state */
        AHCIPORT_SET_STATE(ahci_portp, addrp, 0);

        READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
        SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_COMRESET);
        WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);

        /* PxCMD.FRE should be set before. */
        port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
        ASSERT(port_cmd_status & AHCI_CMD_STATUS_FRE);
        if (!(port_cmd_status & AHCI_CMD_STATUS_FRE))
                return (AHCI_FAILURE);

        /*
         * Give time for COMRESET to percolate, according to the AHCI
         * spec, software shall wait at least 1 millisecond before
         * clearing PxSCTL.DET
         */
        drv_usecwait(AHCI_1MS_USECS*2);

        /*
         * Fetch the SCONTROL again and rewrite the DET part with 0
         * This will generate an Asychronous Notification events.
         */
        READ_PMULT(addrp, SATA_PMULT_REG_SCTL, &port_scontrol, err);
        SCONTROL_SET_DET(port_scontrol, SCONTROL_DET_NOACTION);
        WRITE_PMULT(addrp, SATA_PMULT_REG_SCTL, port_scontrol, err);

        /*
         * The port enters P:StartComm state, and HBA tells link layer to
         * start communication, which involves sending COMRESET to device.
         * And the HBA resets PxTFD.STS to 7Fh.
         *
         * When a COMINIT is received from the device, then the port enters
         * P:ComInit state. And HBA sets PxTFD.STS to FFh or 80h. HBA sets
         * PxSSTS.DET to 1h to indicate a device is detected but communication
         * is not yet established. HBA sets PxSERR.DIAG.X to '1' to indicate
         * a COMINIT has been received.
         */
        /*
         * The DET field is valid only if IPM field indicates
         * that the interface is in active state.
         */
        loop_count = 0;
        do {
                READ_PMULT(addrp, SATA_PMULT_REG_SSTS, &port_sstatus, err);

                if (SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) {
                        /*
                         * If the interface is not active, the DET field
                         * is considered not accurate. So we want to
                         * continue looping.
                         */
                        SSTATUS_SET_DET(port_sstatus, SSTATUS_DET_NODEV);
                }

                if (loop_count++ > AHCI_POLLRATE_PORT_SSTATUS) {
                        /*
                         * We are effectively timing out after 0.1 sec.
                         */
                        break;
                }

                /* Wait for 10 millisec */
                drv_usecwait(AHCI_10MS_USECS);

        } while (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM);

        AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
            "ahci_pmport_reset: 1st loop count: %d, "
            "port_sstatus = 0x%x port %d:%d",
            loop_count, port_sstatus, port, pmport);

        if ((SSTATUS_GET_IPM(port_sstatus) != SSTATUS_IPM_ACTIVE) ||
            (SSTATUS_GET_DET(port_sstatus) != SSTATUS_DET_DEVPRE_PHYCOM)) {
                /*
                 * Either the port is not active or there
                 * is no device present.
                 */
                AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
                    "ahci_pmport_reset: "
                    "no device attached to port %d:%d",
                    port, pmport);
                AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_NONE);
                return (AHCI_SUCCESS);
        }

        /* Now we can make sure there is a device connected to the port */
        /* COMINIT signal is supposed to be received (PxSERR.DIAG.X = '1') */
        READ_PMULT(addrp, SATA_PMULT_REG_SERR, &port_serror, err);

        if (!(port_serror & (1 << 26))) {
                cmn_err(CE_WARN, "!ahci%d: ahci_pmport_reset: "
                    "COMINIT signal from the device not received port %d:%d",
                    instance, port, pmport);

                AHCIPORT_SET_STATE(ahci_portp, addrp, SATA_PSTATE_FAILED);
                return (AHCI_FAILURE);
        }

        /*
         * After clear PxSERR register, we will receive a D2H FIS.
         * Normally this FIS will cause a IPMS error according to AHCI spec
         * v1.2 because there is no command outstanding for it. So we need
         * to ignore this error.
         */
        ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_IGNORE_IPMS;
        WRITE_PMULT(addrp, SATA_PMULT_REG_SERR, AHCI_SERROR_CLEAR_ALL, err);

        /* Now we need to check the D2H FIS by checking IPMS error. */
        loop_count = 0;
        do {
                port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));

                if (loop_count++ > AHCI_POLLRATE_PORT_TFD_ERROR) {
                        /*
                         * No D2H FIS received. This is possible according
                         * to SATA 2.6 spec.
                         */
                        cmn_err(CE_WARN, "ahci_port_reset: port %d:%d "
                            "PxIS.IPMS is not set, we need another "
                            "software reset.", port, pmport);

                        break;
                }

                /* Wait for 10 millisec */
                mutex_exit(&ahci_portp->ahciport_mutex);
                delay(AHCI_10MS_TICKS);
                mutex_enter(&ahci_portp->ahciport_mutex);

        } while (!(port_intr_status & AHCI_INTR_STATUS_IPMS));

        AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
            "ahci_pmport_reset: 2st loop count: %d, "
            "port_sstatus = 0x%x port %d:%d",
            loop_count, port_sstatus, port, pmport);

        /* Clear IPMS */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
            AHCI_INTR_STATUS_IPMS);
        ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;

        /* This pmport is now ready for ahci_tran_start() */
        port_state = AHCIPORT_GET_STATE(ahci_portp, addrp);
        AHCIPORT_SET_STATE(ahci_portp, addrp, port_state|SATA_STATE_READY);

        AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
            "ahci_pmport_reset: succeed at port %d:%d", port, pmport);
        return (AHCI_SUCCESS);

err:    /* R/W PMULT error */
        /* IPMS flags might be set before. */
        ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_IGNORE_IPMS;
        AHCIDBG(AHCIDBG_INFO|AHCIDBG_ERRS, ahci_ctlp,
            "ahci_pmport_reset: failed at port %d:%d", port, pmport);

        return (AHCI_FAILURE);
}

/*
 * AHCI HBA reset ...; the entire HBA is reset, and all ports are disabled.
 * This is the most intrusive.
 *
 * When an HBA reset occurs, Phy communication will be re-established with
 * the device through a COMRESET followed by the normal out-of-band
 * communication sequence defined in Serial ATA. At the end of reset, the
 * device, if working properly, will send a D2H Register FIS, which contains
 * the device signature. When the HBA receives this FIS, it updates PxTFD.STS
 * and PxTFD.ERR register fields, and updates the PxSIG register with the
 * signature.
 *
 * Remember to set GHC.AE to 1 before calling ahci_hba_reset.
 */
static int
ahci_hba_reset(ahci_ctl_t *ahci_ctlp)
{
        ahci_port_t *ahci_portp;
        uint32_t ghc_control;
        uint8_t port;
        int loop_count;
        int rval = AHCI_SUCCESS;

        AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp, "HBA resetting",
            NULL);

        mutex_enter(&ahci_ctlp->ahcictl_mutex);

        ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));

        /* Setting GHC.HR to 1, remember GHC.AE is already set to 1 before */
        ghc_control |= AHCI_HBA_GHC_HR;
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);

        /*
         * Wait until HBA Reset complete or timeout
         */
        loop_count = 0;
        do {
                ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));

                if (loop_count++ > AHCI_POLLRATE_HBA_RESET) {
                        AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                            "ahci hba reset is timing out, "
                            "ghc_control = 0x%x", ghc_control);
                        /* We are effectively timing out after 1 sec. */
                        break;
                }

                /* Wait for 10 millisec */
                drv_usecwait(AHCI_10MS_USECS);
        } while (ghc_control & AHCI_HBA_GHC_HR);

        AHCIDBG(AHCIDBG_POLL_LOOP, ahci_ctlp,
            "ahci_hba_reset: 1st loop count: %d, "
            "ghc_control = 0x%x", loop_count, ghc_control);

        if (ghc_control & AHCI_HBA_GHC_HR) {
                /* The hba is not reset for some reasons */
                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "hba reset failed: HBA in a hung or locked state", NULL);
                mutex_exit(&ahci_ctlp->ahcictl_mutex);
                return (AHCI_FAILURE);
        }

        /*
         * HBA reset will clear (AHCI Spec v1.2 10.4.3) GHC.IE / GHC.AE
         */
        ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));
        ghc_control |= (AHCI_HBA_GHC_AE | AHCI_HBA_GHC_IE);
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);

        mutex_exit(&ahci_ctlp->ahcictl_mutex);

        for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
                /* Only check implemented ports */
                if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
                        continue;
                }

                ahci_portp = ahci_ctlp->ahcictl_ports[port];
                mutex_enter(&ahci_portp->ahciport_mutex);

                /* Make sure the drive is spun-up */
                ahci_staggered_spin_up(ahci_ctlp, port);

                if (ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
                    port, AHCI_PORT_RESET|AHCI_RESET_NO_EVENTS_UP, NULL) !=
                    AHCI_SUCCESS) {
                        rval = AHCI_FAILURE;
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_hba_reset: port %d failed", port);
                        /*
                         * Set the port state to SATA_PSTATE_FAILED if
                         * failed to initialize it.
                         */
                        ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
                }

                mutex_exit(&ahci_portp->ahciport_mutex);
        }

        return (rval);
}

/*
 * This routine is only called from AHCI_ATTACH or phyrdy change
 * case. It first calls software reset, then stop the port and try to
 * read PxSIG register to find the type of device attached to the port.
 *
 * The caller should make sure a valid device exists on specified port and
 * physical communication has been established so that the signature could
 * be retrieved by software reset.
 *
 * NOTE: The port interrupts should be disabled before the function is called.
 */
static void
ahci_find_dev_signature(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    ahci_addr_t *addrp)
{
        ahci_addr_t dev_addr;
        uint32_t signature;
        uint8_t port = addrp->aa_port;
        uint8_t pmport = addrp->aa_pmport;
        int rval;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));
        ASSERT(AHCI_ADDR_IS_VALID(addrp));

        /*
         * If the HBA doesn't support port multiplier, then the driver
         * doesn't need to bother to check port multiplier device.
         *
         * The second port of ICH7 on ASUS P5W DH deluxe motherboard is
         * connected to Silicon Image 4723, to which the two sata drives
         * attached can be set with RAID1, RAID0 or Spanning mode.
         *
         * We found software reset will get failure if port multiplier address
         * 0xf is used by software reset, so just ignore the check since
         * ICH7 doesn't support port multiplier device at all.
         */
        if (AHCI_ADDR_IS_PORT(addrp) &&
            (ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_CBSS)) {
                AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
                    "ahci_find_dev_signature enter: port %d", port);

                /*
                 * NOTE: when the ahci address is a HBA port, we do not know
                 * it is a device or a port multiplier that attached. we need
                 * try a software reset at port multiplier address (0xf
                 * pmport)
                 */
                AHCI_ADDR_SET_PMULT(&dev_addr, addrp->aa_port);
        } else {
                AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
                    "ahci_find_dev_signature enter: port %d:%d",
                    port, pmport);
                dev_addr = *addrp;
        }

        /* Assume it is unknown. */
        AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);

        /* Issue a software reset to get the signature */
        rval = ahci_software_reset(ahci_ctlp, ahci_portp, &dev_addr);
        if (rval != AHCI_SUCCESS) {

                /*
                 * Try to do software reset again with pmport set with 0 if
                 * the controller is set with AHCI_CAP_SRST_NO_HOSTPORT and
                 * the original pmport is set with SATA_PMULT_HOSTPORT (0xf)
                 */
                if ((ahci_ctlp->ahcictl_cap & AHCI_CAP_SRST_NO_HOSTPORT) &&
                    (dev_addr.aa_pmport == SATA_PMULT_HOSTPORT)) {
                        dev_addr.aa_pmport = 0;
                        rval = ahci_software_reset(ahci_ctlp, ahci_portp,
                            &dev_addr);
                }

                if (rval != AHCI_SUCCESS) {
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_find_dev_signature: software reset failed "
                            "at port %d:%d, cannot get signature.",
                            port, pmport);

                        AHCIPORT_SET_STATE(ahci_portp, addrp,
                            SATA_PSTATE_FAILED);
                        return;
                }
        }

        /*
         * ahci_software_reset has started the port, so we need manually stop
         * the port again.
         */
        if (AHCI_ADDR_IS_PORT(addrp)) {
                if (ahci_put_port_into_notrunning_state(ahci_ctlp,
                    ahci_portp, port) != AHCI_SUCCESS) {
                        AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                            "ahci_find_dev_signature: cannot stop port %d.",
                            port);
                        ahci_portp->ahciport_port_state = SATA_PSTATE_FAILED;
                        return;
                }
        }

        /* Now we can make sure that a valid signature is received. */
        signature = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSIG(ahci_ctlp, port));

        if (AHCI_ADDR_IS_PMPORT(addrp)) {
                AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
                    "ahci_find_dev_signature: signature = 0x%x at port %d:%d",
                    signature, port, pmport);
        } else {
                AHCIDBG(AHCIDBG_INIT|AHCIDBG_INFO, ahci_ctlp,
                    "ahci_find_dev_signature: signature = 0x%x at port %d",
                    signature, port);
        }

        /* NOTE: Only support ATAPI device at controller port. */
        if (signature == AHCI_SIGNATURE_ATAPI && !AHCI_ADDR_IS_PORT(addrp))
                signature = SATA_DTYPE_UNKNOWN;

        switch (signature) {

        case AHCI_SIGNATURE_DISK:
                AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATADISK);
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                    "Disk is found at port: %d", port);
                break;

        case AHCI_SIGNATURE_ATAPI:
                AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_ATAPI);
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                    "ATAPI device is found at port: %d", port);
                break;

        case AHCI_SIGNATURE_PORT_MULTIPLIER:
                /* Port Multiplier cannot recursively attached. */
                ASSERT(AHCI_ADDR_IS_PORT(addrp));
                AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_PMULT);
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                    "Port Multiplier is found at port: %d", port);
                break;

        default:
                AHCIPORT_SET_DEV_TYPE(ahci_portp, addrp, SATA_DTYPE_UNKNOWN);
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                    "Unknown device is found at port: %d", port);
        }
}

/*
 * According to the spec, to reliably detect hot plug removals, software
 * must disable interface power management. Software should perform the
 * following initialization on a port after a device is attached:
 *   Set PxSCTL.IPM to 3h or 7h to disable interface state transitions
 *   Set PxCMD.ALPE to '0' to disable aggressive power management
 *   Disable device initiated interface power management by SET FEATURE
 *
 * We can ignore the last item because by default the feature is disabled
 */
static void
ahci_disable_interface_pm(ahci_ctl_t *ahci_ctlp, uint8_t port)
{
        uint32_t port_scontrol, port_cmd_status;
        uint32_t ipm;

        port_scontrol = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port));
        if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SDS) {
                ipm = SCONTROL_IPM_DISABLE_ALL;
        } else {
                ipm = SCONTROL_IPM_DISABLE_BOTH;
        }
        SCONTROL_SET_IPM(port_scontrol, ipm);
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSCTL(ahci_ctlp, port), port_scontrol);

        port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
        port_cmd_status &= ~AHCI_CMD_STATUS_ALPE;
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);
}

/*
 * Start the port - set PxCMD.ST to 1, if PxCMD.FRE is not set
 * to 1, then set it firstly.
 *
 * Each port contains two major DMA engines. One DMA engine walks through
 * the command list, and is controlled by PxCMD.ST. The second DMA engine
 * copies received FISes into system memory, and is controlled by PxCMD.FRE.
 *
 * Software shall not set PxCMD.ST to '1' until it verifies that PxCMD.CR
 * is '0' and has set PxCMD.FRE is '1'. And software shall not clear
 * PxCMD.FRE while PxCMD.ST or PxCMD.CR is set '1'.
 *
 * Software shall not set PxCMD.ST to '1' unless a functional device is
 * present on the port(as determined by PxTFD.STS.BSY = '0',
 * PxTFD.STS.DRQ = '0', and PxSSTS.DET = 3h).
 */
static int
ahci_start_port(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
{
        uint32_t port_cmd_status;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_start_port: %d enter", port);

        if (ahci_portp->ahciport_port_state & SATA_PSTATE_FAILED) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
                    "the state for port %d is 0x%x",
                    port, ahci_portp->ahciport_port_state);
                return (AHCI_FAILURE);
        }

        if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port failed "
                    "no device is attached at port %d", port);
                return (AHCI_FAILURE);
        }

        /* First to set PxCMD.FRE before setting PxCMD.ST. */
        port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));

        if (!(port_cmd_status & AHCI_CMD_STATUS_FRE)) {
                port_cmd_status |= AHCI_CMD_STATUS_FRE;
                ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
                    port_cmd_status);
        }

        port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));

        port_cmd_status |= AHCI_CMD_STATUS_ST;

        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
            port_cmd_status);

        ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_STARTED;

        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_start_port: "
            "PxCMD.ST set to '1' at port %d", port);

        return (AHCI_SUCCESS);
}

/*
 * Setup PxCLB, PxCLBU, PxFB, and PxFBU for particular port. First, we need
 * to make sure PxCMD.ST, PxCMD.CR, PxCMD.FRE, and PxCMD.FR are all cleared.
 * Then set PxCLB, PxCLBU, PxFB, and PxFBU.
 */
static int
ahci_setup_port_base_addresses(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
{
        uint8_t port = ahci_portp->ahciport_port_num;
        uint32_t port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        /* Step 1: Make sure both PxCMD.ST and PxCMD.CR are cleared. */
        if (port_cmd_status & (AHCI_CMD_STATUS_ST | AHCI_CMD_STATUS_CR)) {
                if (ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
                    port) != AHCI_SUCCESS)
                        return (AHCI_FAILURE);

                port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
        }

        /* Step 2: Make sure both PxCMD.FRE and PxCMD.FR are cleared. */
        if (port_cmd_status & (AHCI_CMD_STATUS_FRE | AHCI_CMD_STATUS_FR)) {
                int loop_count = 0;

                /* Clear PxCMD.FRE */
                port_cmd_status &= ~AHCI_CMD_STATUS_FRE;
                ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port),
                    port_cmd_status);

                /* Wait until PxCMD.FR is cleared */
                for (;;) {
                        port_cmd_status =
                            ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));

                        if (!(port_cmd_status & AHCI_CMD_STATUS_FR))
                                break;

                        if (loop_count++ >= AHCI_POLLRATE_PORT_IDLE_FR) {
                                AHCIDBG(AHCIDBG_INIT | AHCIDBG_ERRS, ahci_ctlp,
                                    "ahci_setup_port_base_addresses: cannot "
                                    "clear PxCMD.FR for port %d.", port);

                                /*
                                 * We are effectively timing out after 0.5 sec.
                                 * This value is specified in AHCI spec.
                                 */
                                return (AHCI_FAILURE);
                        }

                        /* Wait for 1 millisec */
                        drv_usecwait(AHCI_1MS_USECS);
                }
        }

        /* Step 3: Config Port Command List Base Address */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCLB(ahci_ctlp, port),
            ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);

        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCLBU(ahci_ctlp, port),
            ahci_portp->ahciport_cmd_list_dma_cookie.dmac_notused);

        /* Step 4: Config Port Received FIS Base Address */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxFB(ahci_ctlp, port),
            ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);

        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxFBU(ahci_ctlp, port),
            ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_notused);

        return (AHCI_SUCCESS);
}

/*
 * Allocate the ahci_port_t including Received FIS and Command List.
 * The argument - port is the physical port number, and not logical
 * port number seen by the SATA framework.
 */
static int
ahci_alloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
{
        dev_info_t *dip = ahci_ctlp->ahcictl_dip;
        ahci_port_t *ahci_portp;
        char taskq_name[64] = "event_handle_taskq";

        ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));

        ahci_portp =
            (ahci_port_t *)kmem_zalloc(sizeof (ahci_port_t), KM_SLEEP);

        ahci_ctlp->ahcictl_ports[port] = ahci_portp;
        ahci_portp->ahciport_port_num = port;

        /* Initialize the port condition variable */
        cv_init(&ahci_portp->ahciport_cv, NULL, CV_DRIVER, NULL);

        /* Initialize the port mutex */
        mutex_init(&ahci_portp->ahciport_mutex, NULL, MUTEX_DRIVER,
            (void *)(uintptr_t)ahci_ctlp->ahcictl_intr_pri);

        mutex_enter(&ahci_portp->ahciport_mutex);

        /*
         * Allocate memory for received FIS structure and
         * command list for this port
         */
        if (ahci_alloc_rcvd_fis(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
                goto err_case1;
        }

        if (ahci_alloc_cmd_list(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
                goto err_case2;
        }

        /* Setup PxCMD.CLB, PxCMD.CLBU, PxCMD.FB, and PxCMD.FBU */
        if (ahci_setup_port_base_addresses(ahci_ctlp, ahci_portp) !=
            AHCI_SUCCESS) {
                goto err_case3;
        }

        (void) snprintf(taskq_name + strlen(taskq_name),
            sizeof (taskq_name) - strlen(taskq_name),
            "_port%d", port);

        /* Create the taskq for the port */
        if ((ahci_portp->ahciport_event_taskq = ddi_taskq_create(dip,
            taskq_name, 2, TASKQ_DEFAULTPRI, 0)) == NULL) {
                cmn_err(CE_WARN, "!ahci%d: ddi_taskq_create failed for event "
                    "handle", ddi_get_instance(ahci_ctlp->ahcictl_dip));
                goto err_case3;
        }

        /* Allocate the argument for the taskq */
        ahci_portp->ahciport_event_args =
            kmem_zalloc(sizeof (ahci_event_arg_t), KM_SLEEP);

        ahci_portp->ahciport_event_args->ahciea_addrp =
            kmem_zalloc(sizeof (ahci_addr_t), KM_SLEEP);

        /* Initialize the done queue */
        ahci_portp->ahciport_doneq = NULL;
        ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
        ahci_portp->ahciport_doneq_len = 0;

        mutex_exit(&ahci_portp->ahciport_mutex);

        return (AHCI_SUCCESS);

err_case3:
        ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);

err_case2:
        ahci_dealloc_rcvd_fis(ahci_portp);

err_case1:
        mutex_exit(&ahci_portp->ahciport_mutex);
        mutex_destroy(&ahci_portp->ahciport_mutex);
        cv_destroy(&ahci_portp->ahciport_cv);

        kmem_free(ahci_portp, sizeof (ahci_port_t));

        return (AHCI_FAILURE);
}

/*
 * Reverse of ahci_alloc_port_state().
 */
static void
ahci_dealloc_port_state(ahci_ctl_t *ahci_ctlp, uint8_t port)
{
        ahci_port_t *ahci_portp = ahci_ctlp->ahcictl_ports[port];

        ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));
        ASSERT(ahci_portp != NULL);

        mutex_enter(&ahci_portp->ahciport_mutex);
        kmem_free(ahci_portp->ahciport_event_args->ahciea_addrp,
            sizeof (ahci_addr_t));
        ahci_portp->ahciport_event_args->ahciea_addrp = NULL;
        kmem_free(ahci_portp->ahciport_event_args, sizeof (ahci_event_arg_t));
        ahci_portp->ahciport_event_args = NULL;
        ddi_taskq_destroy(ahci_portp->ahciport_event_taskq);
        ahci_dealloc_cmd_list(ahci_ctlp, ahci_portp);
        ahci_dealloc_rcvd_fis(ahci_portp);
        ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
        mutex_exit(&ahci_portp->ahciport_mutex);

        mutex_destroy(&ahci_portp->ahciport_mutex);
        cv_destroy(&ahci_portp->ahciport_cv);

        kmem_free(ahci_portp, sizeof (ahci_port_t));

        ahci_ctlp->ahcictl_ports[port] = NULL;
}

/*
 * Allocates memory for the Received FIS Structure
 */
static int
ahci_alloc_rcvd_fis(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
{
        size_t rcvd_fis_size;
        size_t ret_len;
        uint_t cookie_count;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        rcvd_fis_size = sizeof (ahci_rcvd_fis_t);

        /* allocate rcvd FIS dma handle. */
        if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
            &ahci_ctlp->ahcictl_rcvd_fis_dma_attr,
            DDI_DMA_SLEEP,
            NULL,
            &ahci_portp->ahciport_rcvd_fis_dma_handle) !=
            DDI_SUCCESS) {
                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "rcvd FIS dma handle alloc failed", NULL);

                return (AHCI_FAILURE);
        }

        if (ddi_dma_mem_alloc(ahci_portp->ahciport_rcvd_fis_dma_handle,
            rcvd_fis_size,
            &accattr,
            DDI_DMA_CONSISTENT,
            DDI_DMA_SLEEP,
            NULL,
            (caddr_t *)&ahci_portp->ahciport_rcvd_fis,
            &ret_len,
            &ahci_portp->ahciport_rcvd_fis_acc_handle) != 0) {

                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "rcvd FIS dma mem alloc fail", NULL);
                /* error.. free the dma handle. */
                ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
                return (AHCI_FAILURE);
        }

        if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle,
            NULL,
            (caddr_t)ahci_portp->ahciport_rcvd_fis,
            rcvd_fis_size,
            DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
            DDI_DMA_SLEEP,
            NULL,
            &ahci_portp->ahciport_rcvd_fis_dma_cookie,
            &cookie_count) !=  DDI_DMA_MAPPED) {

                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "rcvd FIS dma handle bind fail", NULL);
                /*  error.. free the dma handle & free the memory. */
                ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);
                ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
                return (AHCI_FAILURE);
        }

        bzero((void *)ahci_portp->ahciport_rcvd_fis, rcvd_fis_size);

        AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
            ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_laddress);
        AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
            ahci_portp->ahciport_rcvd_fis_dma_cookie.dmac_address);

        return (AHCI_SUCCESS);
}

/*
 * Deallocates the Received FIS Structure
 */
static void
ahci_dealloc_rcvd_fis(ahci_port_t *ahci_portp)
{
        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        /* Unbind the cmd list dma handle first. */
        (void) ddi_dma_unbind_handle(ahci_portp->ahciport_rcvd_fis_dma_handle);

        /* Then free the underlying memory. */
        ddi_dma_mem_free(&ahci_portp->ahciport_rcvd_fis_acc_handle);

        /* Now free the handle itself. */
        ddi_dma_free_handle(&ahci_portp->ahciport_rcvd_fis_dma_handle);
}

/*
 * Allocates memory for the Command List, which contains up to 32 entries.
 * Each entry contains a command header, which is a 32-byte structure that
 * includes the pointer to the command table.
 */
static int
ahci_alloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
{
        size_t cmd_list_size;
        size_t ret_len;
        uint_t cookie_count;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        cmd_list_size =
            ahci_ctlp->ahcictl_num_cmd_slots * sizeof (ahci_cmd_header_t);

        /* allocate cmd list dma handle. */
        if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
            &ahci_ctlp->ahcictl_cmd_list_dma_attr,
            DDI_DMA_SLEEP,
            NULL,
            &ahci_portp->ahciport_cmd_list_dma_handle) != DDI_SUCCESS) {

                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "cmd list dma handle alloc failed", NULL);
                return (AHCI_FAILURE);
        }

        if (ddi_dma_mem_alloc(ahci_portp->ahciport_cmd_list_dma_handle,
            cmd_list_size,
            &accattr,
            DDI_DMA_CONSISTENT,
            DDI_DMA_SLEEP,
            NULL,
            (caddr_t *)&ahci_portp->ahciport_cmd_list,
            &ret_len,
            &ahci_portp->ahciport_cmd_list_acc_handle) != 0) {

                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "cmd list dma mem alloc fail", NULL);
                /* error.. free the dma handle. */
                ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
                return (AHCI_FAILURE);
        }

        if (ddi_dma_addr_bind_handle(ahci_portp->ahciport_cmd_list_dma_handle,
            NULL,
            (caddr_t)ahci_portp->ahciport_cmd_list,
            cmd_list_size,
            DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
            DDI_DMA_SLEEP,
            NULL,
            &ahci_portp->ahciport_cmd_list_dma_cookie,
            &cookie_count) !=  DDI_DMA_MAPPED) {

                AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                    "cmd list dma handle bind fail", NULL);
                /*  error.. free the dma handle & free the memory. */
                ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);
                ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
                return (AHCI_FAILURE);
        }

        bzero((void *)ahci_portp->ahciport_cmd_list, cmd_list_size);

        AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "64-bit, dma address: 0x%llx",
            ahci_portp->ahciport_cmd_list_dma_cookie.dmac_laddress);

        AHCIDBG(AHCIDBG_INIT, ahci_ctlp, "32-bit, dma address: 0x%x",
            ahci_portp->ahciport_cmd_list_dma_cookie.dmac_address);

        if (ahci_alloc_cmd_tables(ahci_ctlp, ahci_portp) != AHCI_SUCCESS) {
                goto err_out;
        }

        return (AHCI_SUCCESS);

err_out:
        /* Unbind the cmd list dma handle first. */
        (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);

        /* Then free the underlying memory. */
        ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);

        /* Now free the handle itself. */
        ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);

        return (AHCI_FAILURE);
}

/*
 * Deallocates the Command List
 */
static void
ahci_dealloc_cmd_list(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
{
        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        /* First dealloc command table */
        ahci_dealloc_cmd_tables(ahci_ctlp, ahci_portp);

        /* Unbind the cmd list dma handle first. */
        (void) ddi_dma_unbind_handle(ahci_portp->ahciport_cmd_list_dma_handle);

        /* Then free the underlying memory. */
        ddi_dma_mem_free(&ahci_portp->ahciport_cmd_list_acc_handle);

        /* Now free the handle itself. */
        ddi_dma_free_handle(&ahci_portp->ahciport_cmd_list_dma_handle);
}

/*
 * Allocates memory for all Command Tables, which contains Command FIS,
 * ATAPI Command and Physical Region Descriptor Table.
 */
static int
ahci_alloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
{
        size_t ret_len;
        ddi_dma_cookie_t cmd_table_dma_cookie;
        uint_t cookie_count;
        int slot;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        AHCIDBG(AHCIDBG_INIT|AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_alloc_cmd_tables: port %d enter",
            ahci_portp->ahciport_port_num);

        for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
                /* Allocate cmd table dma handle. */
                if (ddi_dma_alloc_handle(ahci_ctlp->ahcictl_dip,
                    &ahci_ctlp->ahcictl_cmd_table_dma_attr,
                    DDI_DMA_SLEEP,
                    NULL,
                    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]) !=
                    DDI_SUCCESS) {

                        AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                            "cmd table dma handle alloc failed", NULL);

                        goto err_out;
                }

                if (ddi_dma_mem_alloc(
                    ahci_portp->ahciport_cmd_tables_dma_handle[slot],
                    ahci_cmd_table_size,
                    &accattr,
                    DDI_DMA_CONSISTENT,
                    DDI_DMA_SLEEP,
                    NULL,
                    (caddr_t *)&ahci_portp->ahciport_cmd_tables[slot],
                    &ret_len,
                    &ahci_portp->ahciport_cmd_tables_acc_handle[slot]) != 0) {

                        AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                            "cmd table dma mem alloc fail", NULL);

                        /* error.. free the dma handle. */
                        ddi_dma_free_handle(
                            &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
                        goto err_out;
                }

                if (ddi_dma_addr_bind_handle(
                    ahci_portp->ahciport_cmd_tables_dma_handle[slot],
                    NULL,
                    (caddr_t)ahci_portp->ahciport_cmd_tables[slot],
                    ahci_cmd_table_size,
                    DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
                    DDI_DMA_SLEEP,
                    NULL,
                    &cmd_table_dma_cookie,
                    &cookie_count) !=  DDI_DMA_MAPPED) {

                        AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                            "cmd table dma handle bind fail", NULL);
                        /*  error.. free the dma handle & free the memory. */
                        ddi_dma_mem_free(
                            &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);
                        ddi_dma_free_handle(
                            &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
                        goto err_out;
                }

                bzero((void *)ahci_portp->ahciport_cmd_tables[slot],
                    ahci_cmd_table_size);

                /* Config Port Command Table Base Address */
                SET_COMMAND_TABLE_BASE_ADDR(
                    (&ahci_portp->ahciport_cmd_list[slot]),
                    cmd_table_dma_cookie.dmac_laddress & 0xffffffffull);

#ifndef __lock_lint
                SET_COMMAND_TABLE_BASE_ADDR_UPPER(
                    (&ahci_portp->ahciport_cmd_list[slot]),
                    cmd_table_dma_cookie.dmac_laddress >> 32);
#endif
        }

        return (AHCI_SUCCESS);
err_out:

        for (slot--; slot >= 0; slot--) {
                /* Unbind the cmd table dma handle first */
                (void) ddi_dma_unbind_handle(
                    ahci_portp->ahciport_cmd_tables_dma_handle[slot]);

                /* Then free the underlying memory */
                ddi_dma_mem_free(
                    &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);

                /* Now free the handle itself */
                ddi_dma_free_handle(
                    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
        }

        return (AHCI_FAILURE);
}

/*
 * Deallocates memory for all Command Tables.
 */
static void
ahci_dealloc_cmd_tables(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp)
{
        int slot;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_dealloc_cmd_tables: %d enter",
            ahci_portp->ahciport_port_num);

        for (slot = 0; slot < ahci_ctlp->ahcictl_num_cmd_slots; slot++) {
                /* Unbind the cmd table dma handle first. */
                (void) ddi_dma_unbind_handle(
                    ahci_portp->ahciport_cmd_tables_dma_handle[slot]);

                /* Then free the underlying memory. */
                ddi_dma_mem_free(
                    &ahci_portp->ahciport_cmd_tables_acc_handle[slot]);

                /* Now free the handle itself. */
                ddi_dma_free_handle(
                    &ahci_portp->ahciport_cmd_tables_dma_handle[slot]);
        }
}

/*
 * Update SATA registers at controller ports
 */
static void
ahci_update_sata_registers(ahci_ctl_t *ahci_ctlp, uint8_t port,
    sata_device_t *sd)
{
        ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));

        sd->satadev_scr.sstatus =
            ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)(AHCI_PORT_PxSSTS(ahci_ctlp, port)));
        sd->satadev_scr.serror =
            ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)(AHCI_PORT_PxSERR(ahci_ctlp, port)));
        sd->satadev_scr.scontrol =
            ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)(AHCI_PORT_PxSCTL(ahci_ctlp, port)));
        sd->satadev_scr.sactive =
            ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)(AHCI_PORT_PxSACT(ahci_ctlp, port)));
}

/*
 * For poll mode, ahci_port_intr will be called to emulate the interrupt
 */
static void
ahci_port_intr(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp, uint8_t port)
{
        uint32_t port_intr_status;
        uint32_t port_intr_enable;

        AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_port_intr enter: port %d", port);

        mutex_enter(&ahci_portp->ahciport_mutex);
        if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_POLLING) {
                /* For SATA_OPMODE_POLLING commands */
                port_intr_enable =
                    (AHCI_INTR_STATUS_DHRS |
                    AHCI_INTR_STATUS_PSS |
                    AHCI_INTR_STATUS_SDBS |
                    AHCI_INTR_STATUS_UFS |
                    AHCI_INTR_STATUS_PCS |
                    AHCI_INTR_STATUS_PRCS |
                    AHCI_INTR_STATUS_OFS |
                    AHCI_INTR_STATUS_INFS |
                    AHCI_INTR_STATUS_IFS |
                    AHCI_INTR_STATUS_HBDS |
                    AHCI_INTR_STATUS_HBFS |
                    AHCI_INTR_STATUS_TFES);
        } else {
                /*
                 * port_intr_enable indicates that the corresponding interrrupt
                 * reporting is enabled.
                 */
                port_intr_enable = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port));
        }

        /* IPMS error in port reset should be ignored according AHCI spec. */
        if (!(ahci_portp->ahciport_flags & AHCI_PORT_FLAG_IGNORE_IPMS))
                port_intr_enable |= AHCI_INTR_STATUS_IPMS;
        mutex_exit(&ahci_portp->ahciport_mutex);

        /*
         * port_intr_stats indicates that the corresponding interrupt
         * condition is active.
         */
        port_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port));

        AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
            "ahci_port_intr: port %d, port_intr_status = 0x%x, "
            "port_intr_enable = 0x%x",
            port, port_intr_status, port_intr_enable);

        port_intr_status &= port_intr_enable;

        /*
         * Pending interrupt events are indicated by the PxIS register.
         * Make sure we don't miss any event.
         */
        if (ahci_check_ctl_handle(ahci_ctlp) != DDI_SUCCESS) {
                ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
                    DDI_SERVICE_UNAFFECTED);
                ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
                    DDI_FME_VERSION);
                return;
        }

        /* First clear the port interrupts status */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
            port_intr_status);

        /* Check the completed non-queued commands */
        if (port_intr_status & (AHCI_INTR_STATUS_DHRS |
            AHCI_INTR_STATUS_PSS)) {
                (void) ahci_intr_cmd_cmplt(ahci_ctlp,
                    ahci_portp, port);
        }

        /* Check the completed queued commands */
        if (port_intr_status & AHCI_INTR_STATUS_SDBS) {
                (void) ahci_intr_set_device_bits(ahci_ctlp,
                    ahci_portp, port);
        }

        /* Check the port connect change status interrupt bit */
        if (port_intr_status & AHCI_INTR_STATUS_PCS) {
                (void) ahci_intr_port_connect_change(ahci_ctlp,
                    ahci_portp, port);
        }

        /* Check the device mechanical presence status interrupt bit */
        if (port_intr_status & AHCI_INTR_STATUS_DMPS) {
                (void) ahci_intr_device_mechanical_presence_status(
                    ahci_ctlp, ahci_portp, port);
        }

        /* Check the PhyRdy change status interrupt bit */
        if (port_intr_status & AHCI_INTR_STATUS_PRCS) {
                (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp,
                    port);
        }

        /*
         * Check the non-fatal error interrupt bits, there are four
         * kinds of non-fatal errors at the time being:
         *
         *    PxIS.UFS - Unknown FIS Error
         *    PxIS.OFS - Overflow Error
         *    PxIS.INFS - Interface Non-Fatal Error
         *    PxIS.IPMS - Incorrect Port Multiplier Status Error
         *
         * For these non-fatal errors, the HBA can continue to operate,
         * so the driver just log the error messages.
         */
        if (port_intr_status & (AHCI_INTR_STATUS_UFS |
            AHCI_INTR_STATUS_OFS |
            AHCI_INTR_STATUS_IPMS |
            AHCI_INTR_STATUS_INFS)) {
                (void) ahci_intr_non_fatal_error(ahci_ctlp, ahci_portp,
                    port, port_intr_status);
        }

        /*
         * Check the fatal error interrupt bits, there are four kinds
         * of fatal errors for AHCI controllers:
         *
         *    PxIS.HBFS - Host Bus Fatal Error
         *    PxIS.HBDS - Host Bus Data Error
         *    PxIS.IFS - Interface Fatal Error
         *    PxIS.TFES - Task File Error
         *
         * The fatal error means the HBA can not recover from it by
         * itself, and it will try to abort the transfer, and the software
         * must intervene to restart the port.
         */
        if (port_intr_status & (AHCI_INTR_STATUS_IFS |
            AHCI_INTR_STATUS_HBDS |
            AHCI_INTR_STATUS_HBFS |
            AHCI_INTR_STATUS_TFES))
                (void) ahci_intr_fatal_error(ahci_ctlp, ahci_portp,
                    port, port_intr_status);

        /* Check the cold port detect interrupt bit */
        if (port_intr_status & AHCI_INTR_STATUS_CPDS) {
                (void) ahci_intr_cold_port_detect(ahci_ctlp, ahci_portp, port);
        }

        /* Second clear the corresponding bit in IS.IPS */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (0x1 << port));

        /* Try to recover at the end of the interrupt handler. */
        if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
            DDI_FM_OK) {
                ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
                    DDI_SERVICE_UNAFFECTED);
                ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
                    DDI_FME_VERSION);
        }
}

/*
 * Interrupt service handler
 */
static uint_t
ahci_intr(caddr_t arg1, caddr_t arg2)
{
#ifndef __lock_lint
        _NOTE(ARGUNUSED(arg2))
#endif
        /* LINTED */
        ahci_ctl_t *ahci_ctlp = (ahci_ctl_t *)arg1;
        ahci_port_t *ahci_portp;
        int32_t global_intr_status;
        uint8_t port;

        /*
         * global_intr_status indicates that the corresponding port has
         * an interrupt pending.
         */
        global_intr_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp));

        if (!(global_intr_status & ahci_ctlp->ahcictl_ports_implemented)) {
                /* The interrupt is not ours */
                return (DDI_INTR_UNCLAIMED);
        }

        /*
         * Check the handle after reading global_intr_status - we don't want
         * to miss any port with pending interrupts.
         */
        if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle) !=
            DDI_FM_OK) {
                ddi_fm_service_impact(ahci_ctlp->ahcictl_dip,
                    DDI_SERVICE_UNAFFECTED);
                ddi_fm_acc_err_clear(ahci_ctlp->ahcictl_ahci_acc_handle,
                    DDI_FME_VERSION);
                return (DDI_INTR_UNCLAIMED);
        }

        /* Loop for all the ports */
        for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
                if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
                        continue;
                }
                if (!((0x1 << port) & global_intr_status)) {
                        continue;
                }

                ahci_portp = ahci_ctlp->ahcictl_ports[port];

                /* Call ahci_port_intr */
                ahci_port_intr(ahci_ctlp, ahci_portp, port);
        }

        return (DDI_INTR_CLAIMED);
}

/*
 * For non-queued commands, when the corresponding bit in the PxCI register
 * is cleared, it means the command is completed successfully. And according
 * to the HBA state machine, there are three conditions which possibly will
 * try to clear the PxCI register bit.
 *      1. Receive one D2H Register FIS which is with 'I' bit set
 *      2. Update PIO Setup FIS
 *      3. Transmit a command and receive R_OK if CTBA.C is set (software reset)
 *
 * Process completed non-queued commands when the interrupt status bit -
 * AHCI_INTR_STATUS_DHRS or AHCI_INTR_STATUS_PSS is set.
 *
 * AHCI_INTR_STATUS_DHRS means a D2H Register FIS has been received
 * with the 'I' bit set. And the following commands will send thus
 * FIS with 'I' bit set upon the successful completion:
 *      1. Non-data commands
 *      2. DMA data-in command
 *      3. DMA data-out command
 *      4. PIO data-out command
 *      5. PACKET non-data commands
 *      6. PACKET PIO data-in command
 *      7. PACKET PIO data-out command
 *      8. PACKET DMA data-in command
 *      9. PACKET DMA data-out command
 *
 * AHCI_INTR_STATUS_PSS means a PIO Setup FIS has been received
 * with the 'I' bit set. And the following commands will send this
 * FIS upon the successful completion:
 *      1. PIO data-in command
 */
static int
ahci_intr_cmd_cmplt(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp, uint8_t port)
{
        uint32_t port_cmd_issue = 0;
        uint32_t finished_tags;
        int finished_slot;
        sata_pkt_t *satapkt;
        ahci_fis_d2h_register_t *rcvd_fisp;
#if AHCI_DEBUG
        ahci_cmd_header_t *cmd_header;
        uint32_t cmd_dmacount;
#endif

        mutex_enter(&ahci_portp->ahciport_mutex);

        if (!ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
            !RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
            !NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                /*
                 * Spurious interrupt. Nothing to be done.
                 */
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (AHCI_SUCCESS);
        }

        port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));

        /* If the PxCI corrupts, don't complete the commmands. */
        if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
            != DDI_FM_OK) {
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (AHCI_FAILURE);
        }

        if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
                /* Slot 0 is always used during error recovery */
                finished_tags = 0x1 & ~port_cmd_issue;
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_intr_cmd_cmplt: port %d the sata pkt for error "
                    "retrieval is finished, and finished_tags = 0x%x",
                    port, finished_tags);
        } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
                finished_tags = 0x1 & ~port_cmd_issue;
                AHCIDBG(AHCIDBG_INFO, ahci_ctlp,
                    "ahci_intr_cmd_cmplt: port %d the sata pkt for r/w "
                    "port multiplier is finished, and finished_tags = 0x%x",
                    port, finished_tags);

        } else {

                finished_tags = ahci_portp->ahciport_pending_tags &
                    ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);
        }

        AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
            "ahci_intr_cmd_cmplt: pending_tags = 0x%x, "
            "port_cmd_issue = 0x%x finished_tags = 0x%x",
            ahci_portp->ahciport_pending_tags, port_cmd_issue,
            finished_tags);

        if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) &&
            (finished_tags == 0x1)) {
                satapkt = ahci_portp->ahciport_err_retri_pkt;
                ASSERT(satapkt != NULL);

                AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
                    "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
                    "with SATA_PKT_COMPLETED", (void *)satapkt);

                ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
                goto out;
        }

        if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) &&
            (finished_tags == 0x1)) {
                satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
                ASSERT(satapkt != NULL);

                AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
                    "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
                    "with SATA_PKT_COMPLETED", (void *)satapkt);

                /* READ PORTMULT need copy out FIS content. */
                if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
                        rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
                            ahcirf_d2h_register_fis);
                        satapkt->satapkt_cmd.satacmd_status_reg =
                            GET_RFIS_STATUS(rcvd_fisp);
                        ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
                }

                ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
                goto out;
        }

        while (finished_tags) {
                finished_slot = ddi_ffs(finished_tags) - 1;
                if (finished_slot == -1) {
                        goto out;
                }

                satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
                ASSERT(satapkt != NULL);
#if AHCI_DEBUG
                /*
                 * For non-native queued commands, the PRD byte count field
                 * shall contain an accurate count of the number of bytes
                 * transferred for the command before the PxCI bit is cleared
                 * to '0' for the command.
                 *
                 * The purpose of this field is to let software know how many
                 * bytes transferred for a given operation in order to
                 * determine if underflow occurred. When issuing native command
                 * queuing commands, this field should not be used and is not
                 * required to be valid since in this case underflow is always
                 * illegal.
                 *
                 * For data reads, the HBA will update its PRD byte count with
                 * the total number of bytes received from the last FIS, and
                 * may be able to continue normally. For data writes, the
                 * device will detect an error, and HBA most likely will get
                 * a fatal error.
                 *
                 * Therefore, here just put code to debug part. And please
                 * refer to the comment above ahci_intr_fatal_error for the
                 * definition of underflow error.
                 */
                cmd_dmacount =
                    ahci_portp->ahciport_prd_bytecounts[finished_slot];
                if (cmd_dmacount) {
                        cmd_header =
                            &ahci_portp->ahciport_cmd_list[finished_slot];
                        AHCIDBG(AHCIDBG_INTR|AHCIDBG_PRDT, ahci_ctlp,
                            "ahci_intr_cmd_cmplt: port %d, "
                            "PRD Byte Count = 0x%x, "
                            "ahciport_prd_bytecounts = 0x%x", port,
                            cmd_header->ahcich_prd_byte_count,
                            cmd_dmacount);

                        if (cmd_header->ahcich_prd_byte_count != cmd_dmacount) {
                                AHCIDBG(AHCIDBG_UNDERFLOW, ahci_ctlp,
                                    "ahci_intr_cmd_cmplt: port %d, "
                                    "an underflow occurred", port);
                        }
                }
#endif

                /*
                 * For SATAC_SMART command with SATA_SMART_RETURN_STATUS
                 * feature, sata_special_regs flag will be set, and the
                 * driver should copy the status and the other corresponding
                 * register values in the D2H Register FIS received (It's
                 * working on Non-data protocol) from the device back to
                 * the sata_cmd.
                 *
                 * For every AHCI port, there is only one Received FIS
                 * structure, which contains the FISes received from the
                 * device, So we're trying to copy the content of D2H
                 * Register FIS in the Received FIS structure back to
                 * the sata_cmd.
                 */
                if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
                        rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
                            ahcirf_d2h_register_fis);
                        satapkt->satapkt_cmd.satacmd_status_reg =
                            GET_RFIS_STATUS(rcvd_fisp);
                        ahci_copy_out_regs(&satapkt->satapkt_cmd, rcvd_fisp);
                }

                AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
                    "ahci_intr_cmd_cmplt: sending up pkt 0x%p "
                    "with SATA_PKT_COMPLETED", (void *)satapkt);

                CLEAR_BIT(ahci_portp->ahciport_pending_tags, finished_slot);
                CLEAR_BIT(finished_tags, finished_slot);
                ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;

                ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
        }
out:
        AHCIDBG(AHCIDBG_PKTCOMP, ahci_ctlp,
            "ahci_intr_cmd_cmplt: pending_tags = 0x%x",
            ahci_portp->ahciport_pending_tags);

        ahci_flush_doneq(ahci_portp);

        mutex_exit(&ahci_portp->ahciport_mutex);

        return (AHCI_SUCCESS);
}

/*
 * AHCI_INTR_STATUS_SDBS means a Set Device Bits FIS has been received
 * with the 'I' bit set and has been copied into system memory. It will
 * be sent under the following situations:
 *
 * 1. NCQ command is completed
 *
 * The completion of NCQ commands (READ/WRITE FPDMA QUEUED) is performed
 * via the Set Device Bits FIS. When such event is generated, the software
 * needs to read PxSACT register and compares the current value to the
 * list of commands previously issue by software. ahciport_pending_ncq_tags
 * keeps the tags of previously issued commands.
 *
 * 2. Asynchronous Notification
 *
 * Asynchronous Notification is a feature in SATA spec 2.6.
 *
 * 1) ATAPI device will send a signal to the host when media is inserted or
 * removed and avoids polling the device for media changes. The signal
 * sent to the host is a Set Device Bits FIS with the 'I' and 'N' bits
 * set to '1'. At the moment, it's not supported yet.
 *
 * 2) Port multiplier will send a signal to the host when a hot plug event
 * has occured on a port multiplier port. It is used when command based
 * switching is employed. This is handled by ahci_intr_pmult_sntf_events()
 */
static int
ahci_intr_set_device_bits(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp, uint8_t port)
{
        ahci_addr_t addr;

        AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
            "ahci_intr_set_device_bits enter: port %d", port);

        /* Initialize HBA port address */
        AHCI_ADDR_SET_PORT(&addr, port);

        /* NCQ plug handler */
        (void) ahci_intr_ncq_events(ahci_ctlp, ahci_portp, &addr);

        /* Check port multiplier's asynchronous notification events */
        if (ahci_ctlp->ahcictl_cap & AHCI_CAP_SNTF) {
                (void) ahci_intr_pmult_sntf_events(ahci_ctlp,
                    ahci_portp, port);
        }

        /* ATAPI events is not supported yet */

        return (AHCI_SUCCESS);
}
/*
 * NCQ interrupt handler. Called upon a NCQ command is completed.
 * Only be called from ahci_intr_set_device_bits().
 */
static int
ahci_intr_ncq_events(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp, ahci_addr_t *addrp)
{
        uint32_t port_sactive;
        uint32_t port_cmd_issue;
        uint32_t issued_tags;
        int issued_slot;
        uint32_t finished_tags;
        int finished_slot;
        uint8_t port = addrp->aa_port;
        sata_pkt_t *satapkt;

        AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
            "ahci_intr_set_device_bits enter: port %d", port);

        mutex_enter(&ahci_portp->ahciport_mutex);
        if (!NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (AHCI_SUCCESS);
        }

        /*
         * First the handler got which commands are finished by checking
         * PxSACT register
         */
        port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));

        finished_tags = ahci_portp->ahciport_pending_ncq_tags &
            ~port_sactive & AHCI_NCQ_SLOT_MASK(ahci_portp);

        AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
            "ahci_intr_set_device_bits: port %d pending_ncq_tags = 0x%x "
            "port_sactive = 0x%x", port,
            ahci_portp->ahciport_pending_ncq_tags, port_sactive);

        AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
            "ahci_intr_set_device_bits: finished_tags = 0x%x", finished_tags);

        /*
         * For NCQ commands, the software can determine which command has
         * already been transmitted to the device by checking PxCI register.
         */
        port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));

        issued_tags = ahci_portp->ahciport_pending_tags &
            ~port_cmd_issue & AHCI_SLOT_MASK(ahci_ctlp);

        /* If the PxSACT/PxCI corrupts, don't complete the NCQ commmands. */
        if (ahci_check_acc_handle(ahci_ctlp->ahcictl_ahci_acc_handle)
            != DDI_FM_OK) {
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (AHCI_FAILURE);
        }

        AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
            "ahci_intr_set_device_bits: port %d pending_tags = 0x%x "
            "port_cmd_issue = 0x%x", port,
            ahci_portp->ahciport_pending_tags, port_cmd_issue);

        AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
            "ahci_intr_set_device_bits: issued_tags = 0x%x", issued_tags);

        /*
         * Clear ahciport_pending_tags bit when the corresponding command
         * is already sent down to the device.
         */
        while (issued_tags) {
                issued_slot = ddi_ffs(issued_tags) - 1;
                if (issued_slot == -1) {
                        goto next;
                }
                CLEAR_BIT(ahci_portp->ahciport_pending_tags, issued_slot);
                CLEAR_BIT(issued_tags, issued_slot);
        }

next:
        while (finished_tags) {
                finished_slot = ddi_ffs(finished_tags) - 1;
                if (finished_slot == -1) {
                        goto out;
                }

                /* The command is certainly transmitted to the device */
                ASSERT(!(ahci_portp->ahciport_pending_tags &
                    (0x1 << finished_slot)));

                satapkt = ahci_portp->ahciport_slot_pkts[finished_slot];
                ASSERT(satapkt != NULL);

                AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ, ahci_ctlp,
                    "ahci_intr_set_device_bits: sending up pkt 0x%p "
                    "with SATA_PKT_COMPLETED", (void *)satapkt);

                CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags, finished_slot);
                CLEAR_BIT(finished_tags, finished_slot);
                ahci_portp->ahciport_slot_pkts[finished_slot] = NULL;

                ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
        }
out:
        AHCIDBG(AHCIDBG_PKTCOMP|AHCIDBG_NCQ, ahci_ctlp,
            "ahci_intr_set_device_bits: port %d "
            "pending_ncq_tags = 0x%x pending_tags = 0x%x",
            port, ahci_portp->ahciport_pending_ncq_tags,
            ahci_portp->ahciport_pending_tags);

        ahci_flush_doneq(ahci_portp);

        mutex_exit(&ahci_portp->ahciport_mutex);

        return (AHCI_SUCCESS);
}

/*
 * Port multiplier asynchronous notification event handler. Called upon a
 * device is hot plugged/pulled.
 *
 * The async-notification event will only be recorded by ahcipmi_snotif_tags
 * here and will be handled by ahci_probe_pmult().
 *
 * NOTE: called only from ahci_port_intr().
 */
static int
ahci_intr_pmult_sntf_events(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    uint8_t port)
{
        sata_device_t sdevice;

        AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
            "ahci_intr_pmult_sntf_events enter: port %d ", port);

        /* no hot-plug while attaching process */
        mutex_enter(&ahci_ctlp->ahcictl_mutex);
        if (ahci_ctlp->ahcictl_flags & AHCI_ATTACH) {
                mutex_exit(&ahci_ctlp->ahcictl_mutex);
                return (AHCI_SUCCESS);
        }
        mutex_exit(&ahci_ctlp->ahcictl_mutex);

        mutex_enter(&ahci_portp->ahciport_mutex);
        if (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (AHCI_SUCCESS);
        }

        ASSERT(ahci_portp->ahciport_pmult_info != NULL);

        ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags =
            ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port));
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSNTF(ahci_ctlp, port),
            AHCI_SNOTIF_CLEAR_ALL);

        if (ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags == 0) {
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (AHCI_SUCCESS);
        }

        /* Port Multiplier sub-device hot-plug handler */
        if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (AHCI_SUCCESS);
        }

        if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_PMULT_SNTF) {
                /* Not allowed to re-enter. */
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (AHCI_SUCCESS);
        }

        ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_PMULT_SNTF;

        /*
         * NOTE:
         * Even if Asynchronous Notification is supported (and enabled) by
         * both controller and the port multiplier, the content of PxSNTF
         * register is always set to 0x8000 by async notification event. We
         * need to check GSCR[32] on the port multiplier to find out the
         * owner of this event.
         * This is not accord with SATA spec 2.6 and needs further
         * clarification.
         */
        /* hot-plug will not reported while reseting. */
        if (ahci_portp->ahciport_reset_in_progress == 1) {
                AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
                    "port %d snotif event ignored", port);
                ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (AHCI_SUCCESS);
        }

        AHCIDBG(AHCIDBG_INFO|AHCIDBG_PMULT, ahci_ctlp,
            "PxSNTF is set to 0x%x by port multiplier",
            ahci_portp->ahciport_pmult_info->ahcipmi_snotif_tags);

        /*
         * Now we need do some necessary operation and inform SATA framework
         * that link/device events has happened.
         */
        bzero((void *)&sdevice, sizeof (sata_device_t));
        sdevice.satadev_addr.cport = ahci_ctlp->
            ahcictl_port_to_cport[port];
        sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
        sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
        sdevice.satadev_state = SATA_PSTATE_PWRON;

        /* Just reject packets, do not stop that port. */
        ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);

        mutex_exit(&ahci_portp->ahciport_mutex);
        sata_hba_event_notify(
            ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
            &sdevice,
            SATA_EVNT_PMULT_LINK_CHANGED);
        mutex_enter(&ahci_portp->ahciport_mutex);

        ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_PMULT_SNTF;
        mutex_exit(&ahci_portp->ahciport_mutex);

        return (AHCI_SUCCESS);
}

/*
 * 1=Change in Current Connect Status. 0=No change in Current Connect Status.
 * This bit reflects the state of PxSERR.DIAG.X. This bit is only cleared
 * when PxSERR.DIAG.X is cleared. When PxSERR.DIAG.X is set to one, it
 * indicates a COMINIT signal was received.
 *
 * Hot plug insertion is detected by reception of a COMINIT signal from the
 * device. On reception of unsolicited COMINIT, the HBA shall generate a
 * COMRESET. If the COMINIT is in responce to a COMRESET, then the HBA shall
 * begin the normal communication negotiation sequence as outlined in the
 * Serial ATA 1.0a specification. When a COMRESET is sent to the device the
 * PxSSTS.DET field shall be cleared to 0h. When a COMINIT is received, the
 * PxSSTS.DET field shall be set to 1h. When the communication negotiation
 * sequence is complete and PhyRdy is true the PxSSTS.DET field shall be set
 * to 3h. Therefore, at the moment the ahci driver is going to check PhyRdy
 * to handle hot plug insertion. In this interrupt handler, just do nothing
 * but print some log message and clear the bit.
 */
static int
ahci_intr_port_connect_change(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp, uint8_t port)
{
#if AHCI_DEBUG
        uint32_t port_serror;
#endif

        mutex_enter(&ahci_portp->ahciport_mutex);

#if AHCI_DEBUG
        port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));

        AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_intr_port_connect_change: port %d, "
            "port_serror = 0x%x", port, port_serror);
#endif

        /* Clear PxSERR.DIAG.X to clear the interrupt bit */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
            SERROR_EXCHANGED_ERR);

        mutex_exit(&ahci_portp->ahciport_mutex);

        return (AHCI_SUCCESS);
}

/*
 * Hot Plug Operation for platforms that support Mechanical Presence
 * Switches.
 *
 * When set, it indicates that a mechanical presence switch attached to this
 * port has been opened or closed, which may lead to a change in the connection
 * state of the device. This bit is only valid if both CAP.SMPS and PxCMD.MPSP
 * are set to '1'.
 *
 * At the moment, this interrupt is not needed and disabled and we just log
 * the debug message.
 */
static int
ahci_intr_device_mechanical_presence_status(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp, uint8_t port)
{
        uint32_t cap_status, port_cmd_status;

        AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_intr_device_mechanical_presence_status enter, "
            "port %d", port);

        cap_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_CAP(ahci_ctlp));

        mutex_enter(&ahci_portp->ahciport_mutex);
        port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));

        if (!(cap_status & AHCI_HBA_CAP_SMPS) ||
            !(port_cmd_status & AHCI_CMD_STATUS_MPSP)) {
                AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
                    "CAP.SMPS or PxCMD.MPSP is not set, so just ignore "
                    "the interrupt: cap_status = 0x%x, "
                    "port_cmd_status = 0x%x", cap_status, port_cmd_status);
                mutex_exit(&ahci_portp->ahciport_mutex);

                return (AHCI_SUCCESS);
        }

#if AHCI_DEBUG
        if (port_cmd_status & AHCI_CMD_STATUS_MPSS) {
                AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
                    "The mechanical presence switch is open: "
                    "port %d, port_cmd_status = 0x%x",
                    port, port_cmd_status);
        } else {
                AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
                    "The mechanical presence switch is close: "
                    "port %d, port_cmd_status = 0x%x",
                    port, port_cmd_status);
        }
#endif

        mutex_exit(&ahci_portp->ahciport_mutex);

        return (AHCI_SUCCESS);
}

/*
 * Native Hot Plug Support.
 *
 * When set, it indicates that the internal PHYRDY signal changed state.
 * This bit reflects the state of PxSERR.DIAG.N.
 *
 * There are three kinds of conditions to generate this interrupt event:
 * 1. a device is inserted
 * 2. a device is disconnected
 * 3. when the link enters/exits a Partial or Slumber interface power
 *    management state
 *
 * If inteface power management is enabled for a port, the PxSERR.DIAG.N
 * bit may be set due to the link entering the Partial or Slumber power
 * management state, rather than due to a hot plug insertion or removal
 * event. So far, the interface power management is disabled, so the
 * driver can reliably get removal detection notification via the
 * PxSERR.DIAG.N bit.
 */
static int
ahci_intr_phyrdy_change(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp, uint8_t port)
{
        uint32_t port_sstatus = 0; /* No dev present & PHY not established. */
        sata_device_t sdevice;
        int dev_exists_now = 0;
        int dev_existed_previously = 0;
        ahci_addr_t port_addr;

        AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_intr_phyrdy_change enter, port %d", port);

        /* Clear PxSERR.DIAG.N to clear the interrupt bit */
        mutex_enter(&ahci_portp->ahciport_mutex);
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
            SERROR_PHY_RDY_CHG);
        mutex_exit(&ahci_portp->ahciport_mutex);

        mutex_enter(&ahci_ctlp->ahcictl_mutex);
        if ((ahci_ctlp->ahcictl_sata_hba_tran == NULL) ||
            (ahci_portp == NULL)) {
                /* The whole controller setup is not yet done. */
                mutex_exit(&ahci_ctlp->ahcictl_mutex);
                return (AHCI_SUCCESS);
        }
        mutex_exit(&ahci_ctlp->ahcictl_mutex);

        mutex_enter(&ahci_portp->ahciport_mutex);

        /* SStatus tells the presence of device. */
        port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));

        if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
                dev_exists_now = 1;
        }

        if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) {
                dev_existed_previously = 1;
        }

        if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_NODEV) {
                ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_NODEV;
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_intr_phyrdy_change: port %d "
                    "AHCI_PORT_FLAG_NODEV is cleared", port);
                if (dev_exists_now == 0)
                        dev_existed_previously = 1;
        }

        bzero((void *)&sdevice, sizeof (sata_device_t));
        sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
        sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
        sdevice.satadev_addr.pmport = 0;
        sdevice.satadev_state = SATA_PSTATE_PWRON;
        ahci_portp->ahciport_port_state = SATA_PSTATE_PWRON;

        AHCI_ADDR_SET_PORT(&port_addr, port);

        ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_HOTPLUG;
        if (dev_exists_now) {
                if (dev_existed_previously) { /* 1 -> 1 */
                        /* Things are fine now. The loss was temporary. */
                        AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
                            "ahci_intr_phyrdy_change  port %d "
                            "device link lost/established", port);

                        mutex_exit(&ahci_portp->ahciport_mutex);
                        sata_hba_event_notify(
                            ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
                            &sdevice,
                            SATA_EVNT_LINK_LOST|SATA_EVNT_LINK_ESTABLISHED);
                        mutex_enter(&ahci_portp->ahciport_mutex);

                } else { /* 0 -> 1 */
                        AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
                            "ahci_intr_phyrdy_change: port %d "
                            "device link established", port);

                        /*
                         * A new device has been detected. The new device
                         * might be a port multiplier instead of a drive, so
                         * we cannot update the signature directly.
                         */
                        (void) ahci_initialize_port(ahci_ctlp,
                            ahci_portp, &port_addr);

                        /* Try to start the port */
                        if (ahci_start_port(ahci_ctlp, ahci_portp, port)
                            != AHCI_SUCCESS) {
                                sdevice.satadev_state |= SATA_PSTATE_FAILED;
                                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                                    "ahci_intr_phyrdy_change: port %d failed "
                                    "at start port", port);
                        }

                        /* Clear the max queue depth for inserted device */
                        ahci_portp->ahciport_max_ncq_tags = 0;

                        mutex_exit(&ahci_portp->ahciport_mutex);
                        sata_hba_event_notify(
                            ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
                            &sdevice,
                            SATA_EVNT_LINK_ESTABLISHED);
                        mutex_enter(&ahci_portp->ahciport_mutex);

                }
        } else { /* No device exists now */

                if (dev_existed_previously) { /* 1 -> 0 */
                        AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
                            "ahci_intr_phyrdy_change: port %d "
                            "device link lost", port);

                        ahci_reject_all_abort_pkts(ahci_ctlp, ahci_portp, port);
                        (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
                            ahci_portp, port);

                        if (ahci_portp->ahciport_device_type ==
                            SATA_DTYPE_PMULT) {
                                ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
                        }

                        /* An existing device is lost. */
                        ahci_portp->ahciport_device_type = SATA_DTYPE_NONE;
                        ahci_portp->ahciport_port_state = SATA_STATE_UNKNOWN;

                        mutex_exit(&ahci_portp->ahciport_mutex);
                        sata_hba_event_notify(
                            ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
                            &sdevice,
                            SATA_EVNT_LINK_LOST);
                        mutex_enter(&ahci_portp->ahciport_mutex);
                }
        }
        ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_HOTPLUG;

        mutex_exit(&ahci_portp->ahciport_mutex);

        return (AHCI_SUCCESS);
}

/*
 * PxIS.UFS - Unknown FIS Error
 *
 * This interrupt event means an unknown FIS was received and has been
 * copied into system memory. An unknown FIS is not considered an illegal
 * FIS, unless the length received is more than 64 bytes. If an unknown
 * FIS arrives with length <= 64 bytes, it is posted and the HBA continues
 * normal operation. If the unknown FIS is more than 64 bytes, then it
 * won't be posted to memory and PxSERR.ERR.P will be set, which is then
 * a fatal error.
 *
 * PxIS.IPMS - Incorrect Port Multiplier Status
 *
 * IPMS Indicates that the HBA received a FIS from a device that did not
 * have a command outstanding. The IPMS bit may be set during enumeration
 * of devices on a Port Multiplier due to the normal Port Multiplier
 * enumeration process. It is recommended that IPMS only be used after
 * enumeration is complete on the Port Multiplier (copied from spec).
 *
 * PxIS.OFS - Overflow Error
 *
 * Command list overflow is defined as software building a command table
 * that has fewer total bytes than the transaction given to the device.
 * On device writes, the HBA will run out of data, and on reads, there
 * will be no room to put the data.
 *
 * For an overflow on data read, either PIO or DMA, the HBA will set
 * PxIS.OFS, and the HBA will do a best effort to continue, and it's a
 * non-fatal error when the HBA can continues. Sometimes, it will cause
 * a fatal error and need the software to do something.
 *
 * For an overflow on data write, setting PxIS.OFS is optional for both
 * DMA and PIO, and it's a fatal error, and a COMRESET is required by
 * software to clean up from this serious error.
 *
 * PxIS.INFS - Interface Non-Fatal Error
 *
 * This interrupt event indicates that the HBA encountered an error on
 * the Serial ATA interface but was able to continue operation. The kind
 * of error usually occurred during a non-Data FIS, and under this condition
 * the FIS will be re-transmitted by HBA automatically.
 *
 * When the FMA is implemented, there should be a stat structure to
 * record how many every kind of error happens.
 */
static int
ahci_intr_non_fatal_error(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    uint8_t port, uint32_t intr_status)
{
        uint32_t port_serror;
#if AHCI_DEBUG
        uint32_t port_cmd_status;
        uint32_t port_cmd_issue;
        uint32_t port_sactive;
        int current_slot;
        uint32_t current_tags;
        sata_pkt_t *satapkt;
        ahci_cmd_header_t *cmd_header;
        uint32_t cmd_dmacount;
#endif

        mutex_enter(&ahci_portp->ahciport_mutex);

        port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));

        AHCIDBG(AHCIDBG_INTR|AHCIDBG_ENTRY|AHCIDBG_ERRS, ahci_ctlp,
            "ahci_intr_non_fatal_error: port %d, "
            "PxSERR = 0x%x, PxIS = 0x%x ", port, port_serror, intr_status);

        ahci_log_serror_message(ahci_ctlp, port, port_serror, 1);

        if (intr_status & AHCI_INTR_STATUS_UFS) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci port %d has unknown FIS error", port);

                /* Clear the interrupt bit by clearing PxSERR.DIAG.F */
                ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
                    SERROR_FIS_TYPE);
        }

#if AHCI_DEBUG
        if (intr_status & AHCI_INTR_STATUS_IPMS) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci port %d "
                    "has Incorrect Port Multiplier Status error", port);
        }

        if (intr_status & AHCI_INTR_STATUS_OFS) {
                AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
                    "ahci port %d has overflow error", port);
        }

        if (intr_status & AHCI_INTR_STATUS_INFS) {
                AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
                    "ahci port %d has interface non fatal error", port);
        }

        /*
         * Record the error occurred command's slot.
         */
        if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
            ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
                port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));

                current_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
                    AHCI_CMD_STATUS_CCS_SHIFT;

                if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
                        satapkt = ahci_portp->ahciport_err_retri_pkt;
                        ASSERT(satapkt != NULL);
                        ASSERT(current_slot == 0);
                } else {
                        satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
                }

                if (satapkt != NULL) {
                        AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_intr_non_fatal_error: pending_tags = 0x%x "
                            "cmd 0x%x", ahci_portp->ahciport_pending_tags,
                            satapkt->satapkt_cmd.satacmd_cmd_reg);

                        AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_intr_non_fatal_error: port %d, "
                            "satapkt 0x%p is being processed when error occurs",
                            port, (void *)satapkt);

                        /*
                         * PRD Byte Count field of command header is not
                         * required to reflect the total number of bytes
                         * transferred when an overflow occurs, so here
                         * just log the value.
                         */
                        cmd_dmacount =
                            ahci_portp->ahciport_prd_bytecounts[current_slot];
                        if (cmd_dmacount) {
                                cmd_header = &ahci_portp->
                                    ahciport_cmd_list[current_slot];
                                AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
                                    "ahci_intr_non_fatal_error: port %d, "
                                    "PRD Byte Count = 0x%x, "
                                    "ahciport_prd_bytecounts = 0x%x", port,
                                    cmd_header->ahcich_prd_byte_count,
                                    cmd_dmacount);
                        }
                }
        } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                /*
                 * For queued command, list those command which have already
                 * been transmitted to the device and still not completed.
                 */
                port_sactive = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));

                port_cmd_issue = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));

                AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_intr_non_fatal_error: pending_ncq_tags = 0x%x "
                    "port_sactive = 0x%x port_cmd_issue = 0x%x",
                    ahci_portp->ahciport_pending_ncq_tags,
                    port_sactive, port_cmd_issue);

                current_tags = ahci_portp->ahciport_pending_ncq_tags &
                    port_sactive & ~port_cmd_issue &
                    AHCI_NCQ_SLOT_MASK(ahci_portp);

                while (current_tags) {
                        current_slot = ddi_ffs(current_tags) - 1;
                        if (current_slot == -1) {
                                goto out;
                        }

                        satapkt = ahci_portp->ahciport_slot_pkts[current_slot];
                        AHCIDBG(AHCIDBG_INTR|AHCIDBG_NCQ|AHCIDBG_ERRS,
                            ahci_ctlp, "ahci_intr_non_fatal_error: "
                            "port %d, satapkt 0x%p is outstanding when "
                            "error occurs", port, (void *)satapkt);

                        CLEAR_BIT(current_tags, current_slot);
                }
        }
out:
#endif
        mutex_exit(&ahci_portp->ahciport_mutex);

        return (AHCI_SUCCESS);
}

/*
 * According to the AHCI spec, the error types include system memory
 * errors, interface errors, port multiplier errors, device errors,
 * command list overflow, command list underflow, native command
 * queuing tag errors and pio data transfer errors.
 *
 * System memory errors such as target abort, master abort, and parity
 * may cause the host to stop, and they are serious errors and needed
 * to be recovered with software intervention. When system software
 * has given a pointer to the HBA that doesn't exist in physical memory,
 * a master/target abort error occurs, and PxIS.HBFS will be set. A
 * data error such as CRC or parity occurs, the HBA aborts the transfer
 * (if necessary) and PxIS.HBDS will be set.
 *
 * Interface errors are errors that occur due to electrical issues on
 * the interface, or protocol miscommunication between the device and
 * HBA, and the respective PxSERR register bit will be set. And PxIS.IFS
 * (fatal) or PxIS.INFS (non-fatal) will be set. The conditions that
 * causes PxIS.IFS/PxIS.INFS to be set are
 *      1. in PxSERR.ERR, P bit is set to '1'
 *      2. in PxSERR.DIAG, C or H bit is set to '1'
 *      3. PhyRdy drop unexpectly, N bit is set to '1'
 * If the error occurred during a non-data FIS, the FIS must be
 * retransmitted, and the error is non-fatal and PxIS.INFS is set. If
 * the error occurred during a data FIS, the transfer will stop, so
 * the error is fatal and PxIS.IFS is set.
 *
 * When a FIS arrives that updates the taskfile, the HBA checks to see
 * if PxTFD.STS.ERR is set. If yes, PxIS.TFES will be set and the HBA
 * stops processing any more commands.
 *
 * Command list overflow is defined as software building a command table
 * that has fewer total bytes than the transaction given to the device.
 * On device writes, the HBA will run out of data, and on reads, there
 * will be no room to put the data. For an overflow on data read, either
 * PIO or DMA, the HBA will set PxIS.OFS, and it's a non-fatal error.
 * For an overflow on data write, setting PxIS.OFS is optional for both
 * DMA and PIO, and a COMRESET is required by software to clean up from
 * this serious error.
 *
 * Command list underflow is defined as software building a command
 * table that has more total bytes than the transaction given to the
 * device. For data writes, both PIO and DMA, the device will detect
 * an error and end the transfer. And these errors are most likely going
 * to be fatal errors that will cause the port to be restarted. For
 * data reads, the HBA updates its PRD byte count, and may be
 * able to continue normally, but is not required to. And The HBA is
 * not required to detect underflow conditions for native command
 * queuing command.
 *
 * The HBA does not actively check incoming DMA Setup FISes to ensure
 * that the PxSACT register bit for that slot is set. Existing error
 * mechanisms, such as host bus failure, or bad protocol, are used to
 * recover from this case.
 *
 * In accordance with Serial ATA 1.0a, DATA FISes prior to the final
 * DATA FIS must be an integral number of Dwords. If the HBA receives
 * a request which is not an integral number of Dwords, the HBA
 * set PxSERR.ERR.P to '1', set PxIS.IFS to '1' and stop running until
 * software restarts the port. And the HBA ensures that the size
 * of the DATA FIS received during a PIO command matches the size in
 * the Transfer Cound field of the preceding PIO Setup FIS, if not, the
 * HBA sets PxSERR.ERR.P to '1', set PxIS.IFS to '1', and then
 * stop running until software restarts the port.
 */
/*
 * the fatal errors include PxIS.IFS, PxIS.HBDS, PxIS.HBFS and PxIS.TFES.
 *
 * PxIS.IFS indicates that the hba encountered an error on the serial ata
 * interface which caused the transfer to stop.
 *
 * PxIS.HBDS indicates that the hba encountered a data error
 * (uncorrectable ecc/parity) when reading from or writing to system memory.
 *
 * PxIS.HBFS indicates that the hba encountered a host bus error that it
 * cannot recover from, such as a bad software pointer.
 *
 * PxIS.TFES is set whenever the status register is updated by the device
 * and the error bit (bit 0) is set.
 */
static int
ahci_intr_fatal_error(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
{
        uint32_t port_cmd_status;
        uint32_t port_serror;
        uint32_t task_file_status;
        int failed_slot;
        sata_pkt_t *spkt = NULL;
        uint8_t err_byte;
        ahci_event_arg_t *args;
        int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
        uint32_t failed_tags = 0;
        int task_fail_flag = 0, task_abort_flag = 0;
        uint32_t slot_status;

        mutex_enter(&ahci_portp->ahciport_mutex);

        /*
         * ahci_intr_phyrdy_change() may have rendered it to
         * SATA_DTYPE_NONE.
         */
        if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
                AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
                    "ahci_intr_fatal_error: port %d no device attached, "
                    "and just return without doing anything", port);
                goto out0;
        }

        if (intr_status & AHCI_INTR_STATUS_TFES) {
                task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));
                AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_intr_fatal_error: port %d "
                    "task_file_status = 0x%x", port, task_file_status);
                task_fail_flag = 1;

                err_byte = (task_file_status & AHCI_TFD_ERR_MASK)
                    >> AHCI_TFD_ERR_SHIFT;
                if (err_byte == SATA_ERROR_ABORT)
                        task_abort_flag = 1;
        }

        /*
         * Here we just log the fatal error info in interrupt context.
         * Misc recovery processing will be handled in task queue.
         */
        if (task_fail_flag  == 1) {
                if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                        /*
                         * Read PxCMD.CCS to determine the slot that the HBA
                         * was processing when the error occurred.
                         */
                        port_cmd_status = ddi_get32(
                            ahci_ctlp->ahcictl_ahci_acc_handle,
                            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
                        failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
                            AHCI_CMD_STATUS_CCS_SHIFT;
                        failed_tags = 0x1 << failed_slot;

                        spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
                        AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_intr_fatal_error: spkt 0x%p is being "
                            "processed when fatal error occurred for port %d",
                            spkt, port);

                        /*
                         * Won't emit the error message if it is an IDENTIFY
                         * DEVICE command sent to an ATAPI device.
                         */
                        if ((spkt != NULL) &&
                            (spkt->satapkt_cmd.satacmd_cmd_reg ==
                            SATAC_ID_DEVICE) &&
                            (task_abort_flag == 1))
                                goto out1;

                        /*
                         * Won't emit the error message if it is an ATAPI PACKET
                         * command
                         */
                        if ((spkt != NULL) &&
                            (spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_PACKET))
                                goto out1;

                } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                        slot_status = ddi_get32(
                            ahci_ctlp->ahcictl_ahci_acc_handle,
                            (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
                        failed_tags = slot_status &
                            AHCI_NCQ_SLOT_MASK(ahci_portp);
                }
        }

        /* print the fatal error type */
        ahci_log_fatal_error_message(ahci_ctlp, port, intr_status);
        ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_ERRPRINT;

        port_serror = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port));

        /* print PxSERR related error message */
        ahci_log_serror_message(ahci_ctlp, port, port_serror, 0);

        /* print task file register value */
        if (task_fail_flag == 1) {
                cmn_err(CE_WARN, "!ahci%d: ahci port %d task_file_status "
                    "= 0x%x", instance, port, task_file_status);
                if (task_abort_flag == 1) {
                        cmn_err(CE_WARN, "!ahci%d: the below command (s) on "
                            "port %d are aborted", instance, port);
                        ahci_dump_commands(ahci_ctlp, port, failed_tags);
                }
        }

out1:
        /* Prepare the argument for the taskq */
        args = ahci_portp->ahciport_event_args;
        args->ahciea_ctlp = (void *)ahci_ctlp;
        args->ahciea_portp = (void *)ahci_portp;
        args->ahciea_event = intr_status;
        AHCI_ADDR_SET_PORT((ahci_addr_t *)args->ahciea_addrp, port);

        /* Start the taskq to handle error recovery */
        if ((ddi_taskq_dispatch(ahci_portp->ahciport_event_taskq,
            ahci_events_handler,
            (void *)args, DDI_NOSLEEP)) != DDI_SUCCESS) {
                ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
                cmn_err(CE_WARN, "!ahci%d: start taskq for error recovery "
                    "port %d failed", instance, port);
        }
out0:
        mutex_exit(&ahci_portp->ahciport_mutex);

        return (AHCI_SUCCESS);
}

/*
 * Hot Plug Operation for platforms that support Cold Presence Detect.
 *
 * When set, a device status has changed as detected by the cold presence
 * detect logic. This bit can either be set due to a non-connected port
 * receiving a device, or a connected port having its device removed.
 * This bit is only valid if the port supports cold presence detect as
 * indicated by PxCMD.CPD set to '1'.
 *
 * At the moment, this interrupt is not needed and disabled and we just
 * log the debug message.
 */
static int
ahci_intr_cold_port_detect(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp, uint8_t port)
{
        uint32_t port_cmd_status;
        sata_device_t sdevice;

        AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
            "ahci_intr_cold_port_detect enter, port %d", port);

        mutex_enter(&ahci_portp->ahciport_mutex);

        port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
        if (!(port_cmd_status & AHCI_CMD_STATUS_CPD)) {
                AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
                    "port %d does not support cold presence detect, so "
                    "we just ignore this interrupt", port);
                mutex_exit(&ahci_portp->ahciport_mutex);
                return (AHCI_SUCCESS);
        }

        AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
            "port %d device status has changed", port);

        bzero((void *)&sdevice, sizeof (sata_device_t));
        sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];
        sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
        sdevice.satadev_addr.pmport = 0;
        sdevice.satadev_state = SATA_PSTATE_PWRON;

        if (port_cmd_status & AHCI_CMD_STATUS_CPS) {
                AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
                    "port %d: a device is hot plugged", port);
                mutex_exit(&ahci_portp->ahciport_mutex);
                sata_hba_event_notify(
                    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
                    &sdevice,
                    SATA_EVNT_DEVICE_ATTACHED);
                mutex_enter(&ahci_portp->ahciport_mutex);

        } else {
                AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
                    "port %d: a device is hot unplugged", port);
                mutex_exit(&ahci_portp->ahciport_mutex);
                sata_hba_event_notify(
                    ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
                    &sdevice,
                    SATA_EVNT_DEVICE_DETACHED);
                mutex_enter(&ahci_portp->ahciport_mutex);
        }

        mutex_exit(&ahci_portp->ahciport_mutex);

        return (AHCI_SUCCESS);
}

/*
 * Enable the interrupts for a particular port.
 */
static void
ahci_enable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
{
        ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_enable_port_intrs enter, port %d", port);

        /*
         * Clear port interrupt status before enabling interrupt
         */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxIS(ahci_ctlp, port),
            AHCI_PORT_INTR_MASK);

        /*
         * Clear the pending bit from IS.IPS
         */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_IS(ahci_ctlp), (1 << port));

        /*
         * Enable the following interrupts:
         *      Device to Host Register FIS Interrupt (DHRS)
         *      PIO Setup FIS Interrupt (PSS)
         *      Set Device Bits Interrupt (SDBS)
         *      Unknown FIS Interrupt (UFS)
         *      Port Connect Change Status (PCS)
         *      PhyRdy Change Status (PRCS)
         *      Overflow Status (OFS)
         *      Interface Non-fatal Error Status (INFS)
         *      Interface Fatal Error Status (IFS)
         *      Host Bus Data Error Status (HBDS)
         *      Host Bus Fatal Error Status (HBFS)
         *      Task File Error Status (TFES)
         */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port),
            (AHCI_INTR_STATUS_DHRS |
            AHCI_INTR_STATUS_PSS |
            AHCI_INTR_STATUS_SDBS |
            AHCI_INTR_STATUS_UFS |
            AHCI_INTR_STATUS_DPS |
            AHCI_INTR_STATUS_PCS |
            AHCI_INTR_STATUS_PRCS |
            AHCI_INTR_STATUS_OFS |
            AHCI_INTR_STATUS_INFS |
            AHCI_INTR_STATUS_IFS |
            AHCI_INTR_STATUS_HBDS |
            AHCI_INTR_STATUS_HBFS |
            AHCI_INTR_STATUS_TFES));
}

/*
 * Enable interrupts for all the ports.
 */
static void
ahci_enable_all_intrs(ahci_ctl_t *ahci_ctlp)
{
        uint32_t ghc_control;

        ASSERT(MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_enable_all_intrs enter", NULL);

        ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));

        ghc_control |= AHCI_HBA_GHC_IE;

        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
}

/*
 * Disable interrupts for a particular port.
 */
static void
ahci_disable_port_intrs(ahci_ctl_t *ahci_ctlp, uint8_t port)
{
        ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
            MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_disable_port_intrs enter, port %d", port);

        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxIE(ahci_ctlp, port), 0);
}

/*
 * Disable interrupts for the whole HBA.
 *
 * The global bit is cleared, then all interrupt sources from all
 * ports are disabled.
 */
static void
ahci_disable_all_intrs(ahci_ctl_t *ahci_ctlp)
{
        uint32_t ghc_control;

        ASSERT(ahci_ctlp->ahcictl_flags & (AHCI_ATTACH | AHCI_QUIESCE) ||
            MUTEX_HELD(&ahci_ctlp->ahcictl_mutex));

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_disable_all_intrs enter",
            NULL);

        ghc_control = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp));

        ghc_control &= ~AHCI_HBA_GHC_IE;

        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_GHC(ahci_ctlp), ghc_control);
}

/*
 * Handle FIXED or MSI interrupts.
 */
/*
 * According to AHCI spec, the HBA may support several interrupt modes:
 *      * pin based interrupts (FIXED)
 *      * single MSI message interrupts
 *      * multiple MSI based message interrupts
 *
 * For pin based interrupts, the software interrupt handler need to check IS
 * register to find out which port has pending interrupts. And then check
 * PxIS register to find out which interrupt events happened on that port.
 *
 * For single MSI message interrupts, MSICAP.MC.MSIE is set with '1', and
 * MSICAP.MC.MME is set with '0'. This mode is similar to pin based interrupts
 * in that software interrupt handler need to check IS register to determine
 * which port triggered the interrupts since it uses a single message for all
 * port interrupts.
 *
 * HBA may optionally support multiple MSI message for better performance. In
 * this mode, each port may have its own interrupt message, and thus generation
 * of interrupts is no longer controlled through the IS register. MSICAP.MC.MMC
 * represents a power-of-2 wrapper on the number of implemented ports, and
 * the mapping of ports to interrupts is done in a 1-1 relationship, up to the
 * maximum number of assigned interrupts. When the number of MSI messages
 * allocated is less than the number requested, then hardware may have two
 * implementation behaviors:
 *      * assign each ports its own interrupt and then force all additional
 *        ports to share the last interrupt message, and this condition is
 *        indicated by clearing GHC.MRSM to '0'
 *      * revert to single MSI mode, indicated by setting GHC.MRSM to '1'
 * When multiple-message MSI is enabled, hardware will still set IS register
 * as single message case. And this IS register may be used by software when
 * fewer than the requested number of messages is granted in order to determine
 * which port had the interrupt.
 *
 * Note: The current ahci driver only supports the first two interrupt modes:
 * pin based interrupts and single MSI message interrupts, and the reason
 * is indicated in below code.
 */
static int
ahci_add_intrs(ahci_ctl_t *ahci_ctlp, int intr_type)
{
        dev_info_t *dip = ahci_ctlp->ahcictl_dip;
        int             count, avail, actual;
        int             i, rc;

        AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INIT|AHCIDBG_INTR, ahci_ctlp,
            "ahci_add_intrs enter interrupt type 0x%x", intr_type);

        /* get number of interrupts. */
        rc = ddi_intr_get_nintrs(dip, intr_type, &count);
        if ((rc != DDI_SUCCESS) || (count == 0)) {
                AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
                    "ddi_intr_get_nintrs() failed, "
                    "rc %d count %d\n", rc, count);
                return (DDI_FAILURE);
        }

        /* get number of available interrupts. */
        rc = ddi_intr_get_navail(dip, intr_type, &avail);
        if ((rc != DDI_SUCCESS) || (avail == 0)) {
                AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
                    "ddi_intr_get_navail() failed, "
                    "rc %d avail %d\n", rc, avail);
                return (DDI_FAILURE);
        }

#if AHCI_DEBUG
        if (avail < count) {
                AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
                    "ddi_intr_get_nintrs returned %d, navail() returned %d",
                    count, avail);
        }
#endif

        /*
         * Note: So far Solaris restricts the maximum number of messages for
         * x86 to 2, that is avail is 2, so here we set the count with 1 to
         * force the driver to use single MSI message interrupt. In future if
         * Solaris remove the restriction, then we need to delete the below
         * code and try to use multiple interrupt routine to gain better
         * performance.
         */
        if ((intr_type == DDI_INTR_TYPE_MSI) && (count > 1)) {
                AHCIDBG(AHCIDBG_INTR, ahci_ctlp,
                    "force to use one interrupt routine though the "
                    "HBA supports %d interrupt", count);
                count = 1;
        }

        /* Allocate an array of interrupt handles. */
        ahci_ctlp->ahcictl_intr_size = count * sizeof (ddi_intr_handle_t);
        ahci_ctlp->ahcictl_intr_htable =
            kmem_alloc(ahci_ctlp->ahcictl_intr_size, KM_SLEEP);

        /* call ddi_intr_alloc(). */
        rc = ddi_intr_alloc(dip, ahci_ctlp->ahcictl_intr_htable,
            intr_type, 0, count, &actual, DDI_INTR_ALLOC_NORMAL);

        if ((rc != DDI_SUCCESS) || (actual == 0)) {
                AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
                    "ddi_intr_alloc() failed, rc %d count %d actual %d "
                    "avail %d\n", rc, count, actual, avail);
                kmem_free(ahci_ctlp->ahcictl_intr_htable,
                    ahci_ctlp->ahcictl_intr_size);
                return (DDI_FAILURE);
        }

        /* use interrupt count returned */
#if AHCI_DEBUG
        if (actual < count) {
                AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
                    "Requested: %d, Received: %d", count, actual);
        }
#endif

        ahci_ctlp->ahcictl_intr_cnt = actual;

        /*
         * Get priority for first, assume remaining are all the same.
         */
        if (ddi_intr_get_pri(ahci_ctlp->ahcictl_intr_htable[0],
            &ahci_ctlp->ahcictl_intr_pri) != DDI_SUCCESS) {
                AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
                    "ddi_intr_get_pri() failed", NULL);

                /* Free already allocated intr. */
                for (i = 0; i < actual; i++) {
                        (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
                }

                kmem_free(ahci_ctlp->ahcictl_intr_htable,
                    ahci_ctlp->ahcictl_intr_size);
                return (DDI_FAILURE);
        }

        /* Test for high level interrupt. */
        if (ahci_ctlp->ahcictl_intr_pri >= ddi_intr_get_hilevel_pri()) {
                AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
                    "ahci_add_intrs: Hi level intr not supported", NULL);

                /* Free already allocated intr. */
                for (i = 0; i < actual; i++) {
                        (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[i]);
                }

                kmem_free(ahci_ctlp->ahcictl_intr_htable,
                    sizeof (ddi_intr_handle_t));

                return (DDI_FAILURE);
        }

        /* Call ddi_intr_add_handler(). */
        for (i = 0; i < actual; i++) {
                if (ddi_intr_add_handler(ahci_ctlp->ahcictl_intr_htable[i],
                    ahci_intr, (caddr_t)ahci_ctlp, NULL) != DDI_SUCCESS) {
                        AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
                            "ddi_intr_add_handler() failed", NULL);

                        /* Free already allocated intr. */
                        for (i = 0; i < actual; i++) {
                                (void) ddi_intr_free(
                                    ahci_ctlp->ahcictl_intr_htable[i]);
                        }

                        kmem_free(ahci_ctlp->ahcictl_intr_htable,
                            ahci_ctlp->ahcictl_intr_size);
                        return (DDI_FAILURE);
                }
        }

        if (ddi_intr_get_cap(ahci_ctlp->ahcictl_intr_htable[0],
            &ahci_ctlp->ahcictl_intr_cap) != DDI_SUCCESS) {
                AHCIDBG(AHCIDBG_INTR|AHCIDBG_INIT, ahci_ctlp,
                    "ddi_intr_get_cap() failed", NULL);

                /* Free already allocated intr. */
                for (i = 0; i < actual; i++) {
                        (void) ddi_intr_free(
                            ahci_ctlp->ahcictl_intr_htable[i]);
                }

                kmem_free(ahci_ctlp->ahcictl_intr_htable,
                    ahci_ctlp->ahcictl_intr_size);
                return (DDI_FAILURE);
        }

        if (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK) {
                /* Call ddi_intr_block_enable() for MSI. */
                (void) ddi_intr_block_enable(ahci_ctlp->ahcictl_intr_htable,
                    ahci_ctlp->ahcictl_intr_cnt);
        } else {
                /* Call ddi_intr_enable() for FIXED or MSI non block enable. */
                for (i = 0; i < ahci_ctlp->ahcictl_intr_cnt; i++) {
                        (void) ddi_intr_enable(
                            ahci_ctlp->ahcictl_intr_htable[i]);
                }
        }

        return (DDI_SUCCESS);
}

/*
 * Removes the registered interrupts irrespective of whether they
 * were legacy or MSI.
 *
 * NOTE: The controller interrupts must be disabled before calling
 * this routine.
 */
static void
ahci_rem_intrs(ahci_ctl_t *ahci_ctlp)
{
        int x;

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp, "ahci_rem_intrs entered", NULL);

        /* Disable all interrupts. */
        if ((ahci_ctlp->ahcictl_intr_type == DDI_INTR_TYPE_MSI) &&
            (ahci_ctlp->ahcictl_intr_cap & DDI_INTR_FLAG_BLOCK)) {
                /* Call ddi_intr_block_disable(). */
                (void) ddi_intr_block_disable(ahci_ctlp->ahcictl_intr_htable,
                    ahci_ctlp->ahcictl_intr_cnt);
        } else {
                for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
                        (void) ddi_intr_disable(
                            ahci_ctlp->ahcictl_intr_htable[x]);
                }
        }

        /* Call ddi_intr_remove_handler(). */
        for (x = 0; x < ahci_ctlp->ahcictl_intr_cnt; x++) {
                (void) ddi_intr_remove_handler(
                    ahci_ctlp->ahcictl_intr_htable[x]);
                (void) ddi_intr_free(ahci_ctlp->ahcictl_intr_htable[x]);
        }

        kmem_free(ahci_ctlp->ahcictl_intr_htable, ahci_ctlp->ahcictl_intr_size);
}

/*
 * This routine tries to put port into P:NotRunning state by clearing
 * PxCMD.ST. HBA will clear PxCI to 0h, PxSACT to 0h, PxCMD.CCS to 0h
 * and PxCMD.CR to '0'.
 */
static int
ahci_put_port_into_notrunning_state(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp, uint8_t port)
{
        uint32_t port_cmd_status;
        int loop_count;

        ASSERT(ahci_ctlp->ahcictl_flags & AHCI_QUIESCE ||
            MUTEX_HELD(&ahci_ctlp->ahcictl_ports[port]->ahciport_mutex));

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_put_port_into_notrunning_state enter: port %d", port);

        port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));

        port_cmd_status &= ~AHCI_CMD_STATUS_ST;
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port), port_cmd_status);

        /* Wait until PxCMD.CR is cleared */
        loop_count = 0;
        do {
                port_cmd_status =
                    ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));

                if (loop_count++ > AHCI_POLLRATE_PORT_IDLE) {
                        AHCIDBG(AHCIDBG_INIT, ahci_ctlp,
                            "clearing port %d CMD.CR timeout, "
                            "port_cmd_status = 0x%x", port,
                            port_cmd_status);
                        /*
                         * We are effectively timing out after 0.5 sec.
                         * This value is specified in AHCI spec.
                         */
                        break;
                }

                /* Wait for 10 millisec */
                drv_usecwait(AHCI_10MS_USECS);
        } while (port_cmd_status & AHCI_CMD_STATUS_CR);

        ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_STARTED;

        if (port_cmd_status & AHCI_CMD_STATUS_CR) {
                AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
                    "ahci_put_port_into_notrunning_state: failed to clear "
                    "PxCMD.CR to '0' after loop count: %d, and "
                    "port_cmd_status = 0x%x", loop_count, port_cmd_status);
                return (AHCI_FAILURE);
        } else {
                AHCIDBG(AHCIDBG_INIT|AHCIDBG_POLL_LOOP, ahci_ctlp,
                    "ahci_put_port_into_notrunning_state: succeeded to clear "
                    "PxCMD.CR to '0' after loop count: %d, and "
                    "port_cmd_status = 0x%x", loop_count, port_cmd_status);
                return (AHCI_SUCCESS);
        }
}

/*
 * First clear PxCMD.ST, and then check PxTFD. If both PxTFD.STS.BSY
 * and PxTFD.STS.DRQ cleared to '0', it means the device is in a
 * stable state, then set PxCMD.ST to '1' to start the port directly.
 * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue a
 * COMRESET to the device to put it in an idle state.
 *
 * The fifth argument returns whether the port reset is involved during
 * the process.
 *
 * The routine will be called under following scenarios:
 *      + To reset the HBA
 *      + To abort the packet(s)
 *      + To reset the port
 *      + To activate the port
 *      + Fatal error recovery
 *      + To abort the timeout packet(s)
 *
 * NOTES!!! During this procedure, PxSERR register will be cleared, and
 * according to the spec, the clearance of three bits will also clear
 * three interrupt status bits.
 *      1. PxSERR.DIAG.F will clear PxIS.UFS
 *      2. PxSERR.DIAG.X will clear PxIS.PCS
 *      3. PxSERR.DIAG.N will clear PxIS.PRCS
 *
 * Among these three interrupt events, the driver needs to take care of
 * PxIS.PRCS, which is the hot plug event. When the driver found out
 * a device was unplugged, it will call the interrupt handler.
 */
static int
ahci_restart_port_wait_till_ready(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp, uint8_t port, int flag, int *reset_flag)
{
        uint32_t port_sstatus;
        uint32_t task_file_status;
        sata_device_t sdevice;
        int rval;
        ahci_addr_t addr_port;
        ahci_pmult_info_t *pminfo = NULL;
        int dev_exists_begin = 0;
        int dev_exists_end = 0;
        uint32_t previous_dev_type = ahci_portp->ahciport_device_type;
        int npmport = 0;
        uint8_t cport = ahci_ctlp->ahcictl_port_to_cport[port];

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_restart_port_wait_till_ready: port %d enter", port);

        AHCI_ADDR_SET_PORT(&addr_port, port);

        if (ahci_portp->ahciport_device_type != SATA_DTYPE_NONE)
                dev_exists_begin = 1;

        /* First clear PxCMD.ST */
        rval = ahci_put_port_into_notrunning_state(ahci_ctlp, ahci_portp,
            port);
        if (rval != AHCI_SUCCESS)
                /*
                 * If PxCMD.CR does not clear within a reasonable time, it
                 * may assume the interface is in a hung condition and may
                 * continue with issuing the port reset.
                 */
                goto reset;

        /* Then clear PxSERR */
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSERR(ahci_ctlp, port),
            AHCI_SERROR_CLEAR_ALL);

        /* Then get PxTFD */
        task_file_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxTFD(ahci_ctlp, port));

        /*
         * Check whether the device is in a stable status, if yes,
         * then start the port directly. However for ahci_tran_reset_dport,
         * we may have to perform a port reset.
         */
        if (!(task_file_status & (AHCI_TFD_STS_BSY | AHCI_TFD_STS_DRQ)) &&
            !(flag & AHCI_PORT_RESET))
                goto out;

reset:
        /*
         * If PxTFD.STS.BSY or PxTFD.STS.DRQ is set to '1', then issue
         * a COMRESET to the device
         */
        ahci_disable_port_intrs(ahci_ctlp, port);
        rval = ahci_port_reset(ahci_ctlp, ahci_portp, &addr_port);
        ahci_enable_port_intrs(ahci_ctlp, port);

#ifdef AHCI_DEBUG
        if (rval != AHCI_SUCCESS)
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_restart_port_wait_till_ready: port %d failed",
                    port);
#endif

        if (reset_flag != NULL)
                *reset_flag = 1;

        /* Indicate to the framework that a reset has happened. */
        if ((ahci_portp->ahciport_device_type != SATA_DTYPE_NONE) &&
            (ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) &&
            !(flag & AHCI_RESET_NO_EVENTS_UP)) {
                /* Set the reset in progress flag */
                ahci_portp->ahciport_reset_in_progress = 1;

                bzero((void *)&sdevice, sizeof (sata_device_t));
                sdevice.satadev_addr.cport =
                    ahci_ctlp->ahcictl_port_to_cport[port];
                sdevice.satadev_addr.pmport = 0;
                sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;

                sdevice.satadev_state = SATA_DSTATE_RESET |
                    SATA_DSTATE_PWR_ACTIVE;
                if (ahci_ctlp->ahcictl_sata_hba_tran) {
                        mutex_exit(&ahci_portp->ahciport_mutex);
                        sata_hba_event_notify(
                            ahci_ctlp->ahcictl_sata_hba_tran->sata_tran_hba_dip,
                            &sdevice,
                            SATA_EVNT_DEVICE_RESET);
                        mutex_enter(&ahci_portp->ahciport_mutex);
                }

                AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
                    "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);
        } else {
                ahci_portp->ahciport_reset_in_progress = 0;
        }

out:
        (void) ahci_start_port(ahci_ctlp, ahci_portp, port);

        /* SStatus tells the presence of device. */
        port_sstatus = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxSSTS(ahci_ctlp, port));

        if (SSTATUS_GET_DET(port_sstatus) == SSTATUS_DET_DEVPRE_PHYCOM) {
                dev_exists_end = 1;
        }

        if (dev_exists_begin == 0 && dev_exists_end == 0) /* 0 -> 0 */
                return (rval);

        /* Check whether a hot plug event happened */
        if (dev_exists_begin == 1 && dev_exists_end == 0) { /* 1 -> 0 */
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_restart_port_wait_till_ready: port %d "
                    "device is removed", port);
                ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_NODEV;
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_restart_port_wait_till_ready: port %d "
                    "AHCI_PORT_FLAG_NODEV flag is set", port);
                mutex_exit(&ahci_portp->ahciport_mutex);
                (void) ahci_intr_phyrdy_change(ahci_ctlp, ahci_portp, port);
                mutex_enter(&ahci_portp->ahciport_mutex);

                return (rval);
        }


        /* 0/1 -> 1 : device may change */
        /*
         * May be called by ahci_fatal_error_recovery_handler, so
         * don't issue software if the previous device is ATAPI.
         */
        if (ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
                return (rval);

        /*
         * The COMRESET will make port multiplier enter legacy mode.
         * Issue a software reset to make it work again.
         */
        ahci_disable_port_intrs(ahci_ctlp, port);
        ahci_find_dev_signature(ahci_ctlp, ahci_portp, &addr_port);
        ahci_enable_port_intrs(ahci_ctlp, port);

        /*
         * Following codes are specific for the port multiplier
         */
        if (previous_dev_type != SATA_DTYPE_PMULT &&
            ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
                /* in case previous_dev_type is corrupt */
                ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
                (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
                return (rval);
        }

        /* Device change: PMult -> Non-PMult */
        if (previous_dev_type == SATA_DTYPE_PMULT &&
            ahci_portp->ahciport_device_type != SATA_DTYPE_PMULT) {
                /*
                 * This might happen because
                 * 1. Software reset failed. Port multiplier is not correctly
                 *    enumerated.
                 * 2. Another non-port-multiplier device is attached. Perhaps
                 *    the port multiplier was replaced by another device by
                 *    whatever reason, but AHCI driver missed hot-plug event.
                 *
                 * Now that the port has been initialized, we just need to
                 * update the port structure according new device, then report
                 * and wait SATA framework to probe new device.
                 */

                /* Force to release pmult resource */
                ahci_dealloc_pmult(ahci_ctlp, ahci_portp);
                (void) ahci_start_port(ahci_ctlp, ahci_portp, port);

                bzero((void *)&sdevice, sizeof (sata_device_t));
                sdevice.satadev_addr.cport =
                    ahci_ctlp->ahcictl_port_to_cport[port];
                sdevice.satadev_addr.pmport = 0;
                sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;

                sdevice.satadev_state = SATA_DSTATE_RESET |
                    SATA_DSTATE_PWR_ACTIVE;

                mutex_exit(&ahci_portp->ahciport_mutex);
                sata_hba_event_notify(
                    ahci_ctlp->ahcictl_dip,
                    &sdevice,
                    SATA_EVNT_DEVICE_RESET);
                mutex_enter(&ahci_portp->ahciport_mutex);

                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "Port multiplier is [Gone] at port %d ", port);
                AHCIDBG(AHCIDBG_EVENT, ahci_ctlp,
                    "port %d sending event up: SATA_EVNT_DEVICE_RESET", port);

                return (AHCI_SUCCESS);
        }

        /* Device change: Non-PMult -> PMult */
        if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {

                /* NOTE: The PxCMD.PMA may be cleared by HBA reset. */
                ahci_alloc_pmult(ahci_ctlp, ahci_portp);

                (void) ahci_start_port(ahci_ctlp, ahci_portp, port);
        }
        pminfo = ahci_portp->ahciport_pmult_info;
        ASSERT(pminfo != NULL);

        /* Device (may) change: PMult -> PMult */
        /*
         * First initialize port multiplier. Set state to READY and wait for
         * probe entry point to initialize it
         */
        ahci_portp->ahciport_port_state = SATA_STATE_READY;

        /*
         * It's a little complicated while target is a port multiplier. we
         * need to COMRESET all pmports behind that PMult otherwise those
         * sub-links between the PMult and the sub-devices will be in an
         * inactive state (indicated by PSCR0/PxSSTS) and the following access
         * to those sub-devices will be rejected by Link-Fatal-Error.
         */
        /*
         * The PxSNTF will be set soon after the pmult is plugged. While the
         * pmult itself is attaching, sata_hba_event_notfiy will fail. so we
         * simply mark every sub-port as 'unknown', then ahci_probe_pmport
         * will initialized it.
         */
        for (npmport = 0; npmport < pminfo->ahcipmi_num_dev_ports; npmport++)
                pminfo->ahcipmi_port_state[npmport] = SATA_STATE_UNKNOWN;

        /* Report reset event. */
        ahci_portp->ahciport_reset_in_progress = 1;

        bzero((void *)&sdevice, sizeof (sata_device_t));
        sdevice.satadev_addr.cport = cport;
        sdevice.satadev_addr.pmport = SATA_PMULT_HOSTPORT;
        sdevice.satadev_addr.qual = SATA_ADDR_PMULT;
        sdevice.satadev_state = SATA_DSTATE_RESET | SATA_DSTATE_PWR_ACTIVE;
        sata_hba_event_notify(ahci_ctlp->ahcictl_dip, &sdevice,
            SATA_EVNT_DEVICE_RESET);

        return (rval);
}

/*
 * This routine may be called under four scenarios:
 *      a) do the recovery from fatal error
 *      b) or we need to timeout some commands
 *      c) or we need to abort some commands
 *      d) or we need reset device/port/controller
 *
 * In all these scenarios, we need to send any pending unfinished
 * commands up to sata framework.
 */
static void
ahci_mop_commands(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp,
    uint32_t slot_status,
    uint32_t failed_tags,
    uint32_t timeout_tags,
    uint32_t aborted_tags,
    uint32_t reset_tags)
{
        uint32_t finished_tags = 0;
        uint32_t unfinished_tags = 0;
        int tmp_slot;
        sata_pkt_t *satapkt;
        int ncq_cmd_in_progress = 0;
        int err_retri_cmd_in_progress = 0;
        int rdwr_pmult_cmd_in_progress = 0;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_mop_commands entered: port: %d slot_status: 0x%x",
            ahci_portp->ahciport_port_num, slot_status);

        AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_mop_commands: failed_tags: 0x%x, "
            "timeout_tags: 0x%x aborted_tags: 0x%x, "
            "reset_tags: 0x%x", failed_tags,
            timeout_tags, aborted_tags, reset_tags);

#ifdef AHCI_DEBUG
        if (ahci_debug_flags & AHCIDBG_ERRS) {
                int i;
                char msg_buf[200] = {0, };
                for (i = 0x1f; i >= 0; i--) {
                        if (ahci_portp->ahciport_slot_pkts[i] != NULL)
                                msg_buf[i] = 'X';
                        else
                                msg_buf[i] = '.';
                }
                msg_buf[0x20] = '\0';
                cmn_err(CE_NOTE, "port[%d] slots: %s",
                    ahci_portp->ahciport_port_num, msg_buf);
                cmn_err(CE_NOTE, "[ERR-RT] %p [RW-PM] %p ",
                    (void *)ahci_portp->ahciport_err_retri_pkt,
                    (void *)ahci_portp->ahciport_rdwr_pmult_pkt);
        }
#endif

        if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                finished_tags = ahci_portp->ahciport_pending_tags &
                    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);

                unfinished_tags = slot_status &
                    AHCI_SLOT_MASK(ahci_ctlp) &
                    ~failed_tags &
                    ~aborted_tags &
                    ~reset_tags &
                    ~timeout_tags;
        } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                ncq_cmd_in_progress = 1;
                finished_tags = ahci_portp->ahciport_pending_ncq_tags &
                    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);

                unfinished_tags = slot_status &
                    AHCI_NCQ_SLOT_MASK(ahci_portp) &
                    ~failed_tags &
                    ~aborted_tags &
                    ~reset_tags &
                    ~timeout_tags;
        } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {

                /*
                 * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT is
                 * set, it means REQUEST SENSE or READ LOG EXT command doesn't
                 * complete successfully due to one of the following three
                 * conditions:
                 *
                 *      1. Fatal error - failed_tags includes its slot
                 *      2. Timed out - timeout_tags includes its slot
                 *      3. Aborted when hot unplug - aborted_tags includes its
                 *         slot
                 *
                 * Please note that the command is always sent down in Slot 0
                 */
                err_retri_cmd_in_progress = 1;
                AHCIDBG(AHCIDBG_ERRS|AHCIDBG_NCQ, ahci_ctlp,
                    "ahci_mop_commands is called for port %d while "
                    "REQUEST SENSE or READ LOG EXT for error retrieval "
                    "is being executed slot_status = 0x%x",
                    ahci_portp->ahciport_port_num, slot_status);
                ASSERT(ahci_portp->ahciport_mop_in_progress > 1);
                ASSERT(slot_status == 0x1);
        } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
                rdwr_pmult_cmd_in_progress = 1;
                AHCIDBG(AHCIDBG_ERRS|AHCIDBG_PMULT, ahci_ctlp,
                    "ahci_mop_commands is called for port %d while "
                    "READ/WRITE PORTMULT command is being executed",
                    ahci_portp->ahciport_port_num);

                ASSERT(slot_status == 0x1);
        }

#ifdef AHCI_DEBUG
        AHCIDBG(AHCIDBG_ERRS|AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_mop_commands: finished_tags: 0x%x, "
            "unfinished_tags 0x%x", finished_tags, unfinished_tags);
#endif

        /* Send up finished packets with SATA_PKT_COMPLETED */
        while (finished_tags) {
                tmp_slot = ddi_ffs(finished_tags) - 1;
                if (tmp_slot == -1) {
                        break;
                }

                satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
                ASSERT(satapkt != NULL);

                AHCIDBG(AHCIDBG_INFO, ahci_ctlp, "ahci_mop_commands: "
                    "sending up pkt 0x%p with SATA_PKT_COMPLETED",
                    (void *)satapkt);

                /*
                 * Cannot fetch the return register content since the port
                 * was restarted, so the corresponding tag will be set to
                 * aborted tags.
                 */
                if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
                        CLEAR_BIT(finished_tags, tmp_slot);
                        aborted_tags |= tmp_slot;
                        continue;
                }

                if (ncq_cmd_in_progress)
                        CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
                            tmp_slot);
                CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
                CLEAR_BIT(finished_tags, tmp_slot);
                ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;

                ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_COMPLETED);
        }

        /* Send up failed packets with SATA_PKT_DEV_ERROR. */
        while (failed_tags) {
                if (err_retri_cmd_in_progress) {
                        satapkt = ahci_portp->ahciport_err_retri_pkt;
                        ASSERT(satapkt != NULL);
                        ASSERT(failed_tags == 0x1);

                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
                            "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
                            (void *)satapkt);
                        ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
                        break;
                }
                if (rdwr_pmult_cmd_in_progress) {
                        satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
                        ASSERT(satapkt != NULL);
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_mop_commands: sending up "
                            "rdwr pmult pkt 0x%p with SATA_PKT_DEV_ERROR",
                            (void *)satapkt);
                        ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
                        break;
                }

                tmp_slot = ddi_ffs(failed_tags) - 1;
                if (tmp_slot == -1) {
                        break;
                }

                satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
                ASSERT(satapkt != NULL);

                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
                    "sending up pkt 0x%p with SATA_PKT_DEV_ERROR",
                    (void *)satapkt);

                if (ncq_cmd_in_progress)
                        CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
                            tmp_slot);
                CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
                CLEAR_BIT(failed_tags, tmp_slot);
                ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;

                ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_DEV_ERROR);
        }

        /* Send up timeout packets with SATA_PKT_TIMEOUT. */
        while (timeout_tags) {
                if (err_retri_cmd_in_progress) {
                        satapkt = ahci_portp->ahciport_err_retri_pkt;
                        ASSERT(satapkt != NULL);
                        ASSERT(timeout_tags == 0x1);

                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
                            "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
                            (void *)satapkt);
                        ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
                        break;
                }
                if (rdwr_pmult_cmd_in_progress) {
                        satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
                        ASSERT(satapkt != NULL);
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_mop_commands: sending up "
                            "rdwr pmult pkt 0x%p with SATA_PKT_TIMEOUT",
                            (void *)satapkt);
                        ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
                        break;
                }

                tmp_slot = ddi_ffs(timeout_tags) - 1;
                if (tmp_slot == -1) {
                        break;
                }

                satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
                ASSERT(satapkt != NULL);

                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
                    "sending up pkt 0x%p with SATA_PKT_TIMEOUT",
                    (void *)satapkt);

                if (ncq_cmd_in_progress)
                        CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
                            tmp_slot);
                CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
                CLEAR_BIT(timeout_tags, tmp_slot);
                ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;

                ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_TIMEOUT);
        }

        /* Send up aborted packets with SATA_PKT_ABORTED */
        while (aborted_tags) {
                if (err_retri_cmd_in_progress) {
                        satapkt = ahci_portp->ahciport_err_retri_pkt;
                        ASSERT(satapkt != NULL);
                        ASSERT(aborted_tags == 0x1);

                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
                            "sending up pkt 0x%p with SATA_PKT_ABORTED",
                            (void *)satapkt);
                        ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
                        break;
                }
                if (rdwr_pmult_cmd_in_progress) {
                        satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
                        ASSERT(satapkt != NULL);
                        ASSERT(aborted_tags == 0x1);
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_mop_commands: sending up "
                            "rdwr pmult pkt 0x%p with SATA_PKT_ABORTED",
                            (void *)satapkt);
                        ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
                        break;
                }

                tmp_slot = ddi_ffs(aborted_tags) - 1;
                if (tmp_slot == -1) {
                        break;
                }

                satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
                ASSERT(satapkt != NULL);

                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
                    "sending up pkt 0x%p with SATA_PKT_ABORTED",
                    (void *)satapkt);

                if (ncq_cmd_in_progress)
                        CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
                            tmp_slot);
                CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
                CLEAR_BIT(aborted_tags, tmp_slot);
                ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;

                ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_ABORTED);
        }

        /* Send up reset packets with SATA_PKT_RESET. */
        while (reset_tags) {
                if (rdwr_pmult_cmd_in_progress) {
                        satapkt = ahci_portp->ahciport_rdwr_pmult_pkt;
                        ASSERT(satapkt != NULL);
                        ASSERT(aborted_tags == 0x1);
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_mop_commands: sending up "
                            "rdwr pmult pkt 0x%p with SATA_PKT_RESET",
                            (void *)satapkt);
                        ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
                        break;
                }

                tmp_slot = ddi_ffs(reset_tags) - 1;
                if (tmp_slot == -1) {
                        break;
                }

                satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
                ASSERT(satapkt != NULL);

                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
                    "sending up pkt 0x%p with SATA_PKT_RESET",
                    (void *)satapkt);

                if (ncq_cmd_in_progress)
                        CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
                            tmp_slot);
                CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
                CLEAR_BIT(reset_tags, tmp_slot);
                ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;

                ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
        }

        /* Send up unfinished packets with SATA_PKT_RESET */
        while (unfinished_tags) {
                tmp_slot = ddi_ffs(unfinished_tags) - 1;
                if (tmp_slot == -1) {
                        break;
                }

                satapkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
                ASSERT(satapkt != NULL);

                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, "ahci_mop_commands: "
                    "sending up pkt 0x%p with SATA_PKT_RESET",
                    (void *)satapkt);

                if (ncq_cmd_in_progress)
                        CLEAR_BIT(ahci_portp->ahciport_pending_ncq_tags,
                            tmp_slot);
                CLEAR_BIT(ahci_portp->ahciport_pending_tags, tmp_slot);
                CLEAR_BIT(unfinished_tags, tmp_slot);
                ahci_portp->ahciport_slot_pkts[tmp_slot] = NULL;

                ahci_add_doneq(ahci_portp, satapkt, SATA_PKT_RESET);
        }

        ahci_portp->ahciport_mop_in_progress--;
        ASSERT(ahci_portp->ahciport_mop_in_progress >= 0);

        if (ahci_portp->ahciport_mop_in_progress == 0)
                ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_MOPPING;

        ahci_flush_doneq(ahci_portp);
}

/*
 * This routine is going to first request a READ LOG EXT sata pkt from sata
 * module, and then deliver it to the HBA to get the ncq failure context.
 * The return value is the exactly failed tags.
 */
static uint32_t
ahci_get_rdlogext_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    uint8_t port)
{
        sata_device_t   sdevice;
        sata_pkt_t      *rdlog_spkt, *spkt;
        ddi_dma_handle_t buf_dma_handle;
        ahci_addr_t     addr;
        int             loop_count;
        int             rval;
        int             failed_slot;
        uint32_t        failed_tags = 0;
        struct sata_ncq_error_recovery_page *ncq_err_page;

        AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_NCQ, ahci_ctlp,
            "ahci_get_rdlogext_data enter: port %d", port);

        /* Prepare the sdevice data */
        bzero((void *)&sdevice, sizeof (sata_device_t));
        sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];

        sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
        sdevice.satadev_addr.pmport = 0;

        /* Translate sata_device.satadev_addr -> ahci_addr */
        ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);

        /*
         * Call the sata hba interface to get a rdlog spkt
         */
        loop_count = 0;
loop:
        rdlog_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
            &sdevice, SATA_ERR_RETR_PKT_TYPE_NCQ);
        if (rdlog_spkt == NULL) {
                if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
                        /* Sleep for a while */
                        drv_usecwait(AHCI_10MS_USECS);
                        goto loop;
                }
                /* Timed out after 1s */
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "failed to get rdlog spkt for port %d", port);
                return (failed_tags);
        }

        ASSERT(rdlog_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);

        /*
         * This flag is used to handle the specific error recovery when the
         * READ LOG EXT command gets a failure (fatal error or time-out).
         */
        ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RDLOGEXT;

        /*
         * This start is not supposed to fail because after port is restarted,
         * the whole command list is empty.
         */
        ahci_portp->ahciport_err_retri_pkt = rdlog_spkt;
        (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rdlog_spkt);
        ahci_portp->ahciport_err_retri_pkt = NULL;

        /* Remove the flag after READ LOG EXT command is completed */
        ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RDLOGEXT;

        if (rdlog_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
                /* Update the request log data */
                buf_dma_handle = *(ddi_dma_handle_t *)
                    (rdlog_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
                rval = ddi_dma_sync(buf_dma_handle, 0, 0,
                    DDI_DMA_SYNC_FORKERNEL);
                if (rval == DDI_SUCCESS) {
                        ncq_err_page =
                            (struct sata_ncq_error_recovery_page *)rdlog_spkt->
                            satapkt_cmd.satacmd_bp->b_un.b_addr;

                        /* Get the failed tag */
                        failed_slot = ncq_err_page->ncq_tag;
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_get_rdlogext_data: port %d "
                            "failed slot %d", port, failed_slot);
                        if (failed_slot & NQ) {
                                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                                    "the failed slot is not a valid tag", NULL);
                                goto out;
                        }

                        failed_slot &= NCQ_TAG_MASK;
                        spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
                        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_get_rdlogext_data: failed spkt 0x%p",
                            (void *)spkt);
                        if (spkt == NULL) {
                                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                                    "the failed slot spkt is NULL", NULL);
                                goto out;
                        }

                        failed_tags = 0x1 << failed_slot;

                        /* Fill out the error context */
                        ahci_copy_ncq_err_page(&spkt->satapkt_cmd,
                            ncq_err_page);
                        ahci_update_sata_registers(ahci_ctlp, port,
                            &spkt->satapkt_device);
                }
        }
out:
        sata_free_error_retrieval_pkt(rdlog_spkt);

        return (failed_tags);
}

/*
 * This routine is going to first request a REQUEST SENSE sata pkt from sata
 * module, and then deliver it to the HBA to get the sense data and copy
 * the sense data back to the orignal failed sata pkt, and free the REQUEST
 * SENSE sata pkt later.
 */
static void
ahci_get_rqsense_data(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    uint8_t port, sata_pkt_t *spkt)
{
        sata_device_t   sdevice;
        sata_pkt_t      *rs_spkt;
        sata_cmd_t      *sata_cmd;
        ddi_dma_handle_t buf_dma_handle;
        ahci_addr_t     addr;
        int             loop_count;
#if AHCI_DEBUG
        struct scsi_extended_sense *rqsense;
#endif

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_get_rqsense_data enter: port %d", port);

        /* Prepare the sdevice data */
        bzero((void *)&sdevice, sizeof (sata_device_t));
        sdevice.satadev_addr.cport = ahci_ctlp->ahcictl_port_to_cport[port];

        sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
        sdevice.satadev_addr.pmport = 0;

        /* Translate sata_device.satadev_addr -> ahci_addr */
        ahci_get_ahci_addr(ahci_ctlp, &sdevice, &addr);

        sata_cmd = &spkt->satapkt_cmd;

        /*
         * Call the sata hba interface to get a rs spkt
         */
        loop_count = 0;
loop:
        rs_spkt = sata_get_error_retrieval_pkt(ahci_ctlp->ahcictl_dip,
            &sdevice, SATA_ERR_RETR_PKT_TYPE_ATAPI);
        if (rs_spkt == NULL) {
                if (loop_count++ < AHCI_POLLRATE_GET_SPKT) {
                        /* Sleep for a while */
                        drv_usecwait(AHCI_10MS_USECS);
                        goto loop;

                }
                /* Timed out after 1s */
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "failed to get rs spkt for port %d", port);
                return;
        }

        ASSERT(rs_spkt->satapkt_op_mode & SATA_OPMODE_SYNCH);

        /*
         * This flag is used to handle the specific error recovery when the
         * REQUEST SENSE command gets a faiure (fatal error or time-out).
         */
        ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_RQSENSE;

        /*
         * This start is not supposed to fail because after port is restarted,
         * the whole command list is empty.
         */
        ahci_portp->ahciport_err_retri_pkt = rs_spkt;
        (void) ahci_do_sync_start(ahci_ctlp, ahci_portp, &addr, rs_spkt);
        ahci_portp->ahciport_err_retri_pkt = NULL;

        /* Remove the flag after REQUEST SENSE command is completed */
        ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_RQSENSE;

        if (rs_spkt->satapkt_reason == SATA_PKT_COMPLETED) {
                /* Update the request sense data */
                buf_dma_handle = *(ddi_dma_handle_t *)
                    (rs_spkt->satapkt_cmd.satacmd_err_ret_buf_handle);
                (void) ddi_dma_sync(buf_dma_handle, 0, 0,
                    DDI_DMA_SYNC_FORKERNEL);
                /* Copy the request sense data */
                bcopy(rs_spkt->
                    satapkt_cmd.satacmd_bp->b_un.b_addr,
                    &sata_cmd->satacmd_rqsense,
                    SATA_ATAPI_MIN_RQSENSE_LEN);
#if AHCI_DEBUG
                rqsense = (struct scsi_extended_sense *)
                    sata_cmd->satacmd_rqsense;

                /* Dump the sense data */
                AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp, "\n", NULL);
                AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
                    "Sense data for satapkt %p ATAPI cmd 0x%x",
                    spkt, sata_cmd->satacmd_acdb[0]);
                AHCIDBG(AHCIDBG_SENSEDATA, ahci_ctlp,
                    "  es_code 0x%x es_class 0x%x "
                    "es_key 0x%x es_add_code 0x%x "
                    "es_qual_code 0x%x",
                    rqsense->es_code, rqsense->es_class,
                    rqsense->es_key, rqsense->es_add_code,
                    rqsense->es_qual_code);
#endif
        }

        sata_free_error_retrieval_pkt(rs_spkt);
}

/*
 * Fatal errors will cause the HBA to enter the ERR: Fatal state. To recover,
 * the port must be restarted. When the HBA detects thus error, it may try
 * to abort a transfer. And if the transfer was aborted, the device is
 * expected to send a D2H Register FIS with PxTFD.STS.ERR set to '1' and both
 * PxTFD.STS.BSY and PxTFD.STS.DRQ cleared to '0'. Then system software knows
 * that the device is in a stable status and transfers may be restarted without
 * issuing a COMRESET to the device. If PxTFD.STS.BSY or PxTFD.STS.DRQ is set,
 * then the software will send the COMRESET to do the port reset.
 *
 * Software should perform the appropriate error recovery actions based on
 * whether non-queued commands were being issued or natived command queuing
 * commands were being issued.
 *
 * And software will complete the command that had the error with error mark
 * to higher level software.
 *
 * Fatal errors include the following:
 *      PxIS.IFS - Interface Fatal Error Status
 *      PxIS.HBDS - Host Bus Data Error Status
 *      PxIS.HBFS - Host Bus Fatal Error Status
 *      PxIS.TFES - Task File Error Status
 */
static void
ahci_fatal_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp, ahci_addr_t *addrp, uint32_t intr_status)
{
        uint32_t        port_cmd_status;
        uint32_t        slot_status = 0;
        uint32_t        failed_tags = 0;
        int             failed_slot;
        int             reset_flag = 0, flag = 0;
        ahci_fis_d2h_register_t *ahci_rcvd_fisp;
        sata_cmd_t      *sata_cmd = NULL;
        sata_pkt_t      *spkt = NULL;
#if AHCI_DEBUG
        ahci_cmd_header_t *cmd_header;
#endif
        uint8_t         port = addrp->aa_port;
        int             instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);
        int             rval;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_fatal_error_recovery_handler enter: port %d", port);

        /* Port multiplier error */
        if (ahci_portp->ahciport_device_type == SATA_DTYPE_PMULT) {
                /* FBS code is neither completed nor tested. */
                ahci_pmult_error_recovery_handler(ahci_ctlp, ahci_portp,
                    port, intr_status);

                /* Force a port reset */
                flag = AHCI_PORT_RESET;
        }

        if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
            ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {

                /* Read PxCI to see which commands are still outstanding */
                slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));

                /*
                 * Read PxCMD.CCS to determine the slot that the HBA
                 * was processing when the error occurred.
                 */
                port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));
                failed_slot = (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
                    AHCI_CMD_STATUS_CCS_SHIFT;

                if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
                        spkt = ahci_portp->ahciport_err_retri_pkt;
                        ASSERT(spkt != NULL);
                } else {
                        spkt = ahci_portp->ahciport_slot_pkts[failed_slot];
                        if (spkt == NULL) {
                                /* May happen when interface errors occur? */
                                goto next;
                        }
                }

#if AHCI_DEBUG
                /*
                 * Debugging purpose...
                 */
                if (ahci_portp->ahciport_prd_bytecounts[failed_slot]) {
                        cmd_header =
                            &ahci_portp->ahciport_cmd_list[failed_slot];
                        AHCIDBG(AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
                            "ahci_fatal_error_recovery_handler: port %d, "
                            "PRD Byte Count = 0x%x, "
                            "ahciport_prd_bytecounts = 0x%x", port,
                            cmd_header->ahcich_prd_byte_count,
                            ahci_portp->ahciport_prd_bytecounts[failed_slot]);
                }
#endif

                sata_cmd = &spkt->satapkt_cmd;

                /* Fill out the status and error registers for PxIS.TFES */
                if (intr_status & AHCI_INTR_STATUS_TFES) {
                        ahci_rcvd_fisp = &(ahci_portp->ahciport_rcvd_fis->
                            ahcirf_d2h_register_fis);

                        /* Copy the error context back to the sata_cmd */
                        ahci_copy_err_cnxt(sata_cmd, ahci_rcvd_fisp);
                }

                /* The failed command must be one of the outstanding commands */
                failed_tags = 0x1 << failed_slot;
                ASSERT(failed_tags & slot_status);

                /* Update the sata registers, especially PxSERR register */
                ahci_update_sata_registers(ahci_ctlp, port,
                    &spkt->satapkt_device);

        } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                /* Read PxSACT to see which commands are still outstanding */
                slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
        }
next:

#if AHCI_DEBUG
        /*
         * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
         * set, it means a fatal error happened after REQUEST SENSE command
         * or READ LOG EXT command is delivered to the HBA during the error
         * recovery process. At this time, the only outstanding command is
         * supposed to be REQUEST SENSE command or READ LOG EXT command.
         */
        if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "ahci_fatal_error_recovery_handler: port %d REQUEST SENSE "
                    "command or READ LOG EXT command for error data retrieval "
                    "failed", port);
                ASSERT(slot_status == 0x1);
                ASSERT(failed_slot == 0);
                ASSERT(spkt->satapkt_cmd.satacmd_acdb[0] ==
                    SCMD_REQUEST_SENSE ||
                    spkt->satapkt_cmd.satacmd_cmd_reg ==
                    SATAC_READ_LOG_EXT);
        }
#endif

        ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
        ahci_portp->ahciport_mop_in_progress++;

        rval = ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
            port, flag, &reset_flag);

        if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
                ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
                if (rval == AHCI_SUCCESS)
                        cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
                            "succeed", instance, port);
                else
                        cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
                            "failed", instance, port);
        }

        /*
         * Won't retrieve error information:
         * 1. Port reset was involved to recover
         * 2. Device is gone
         * 3. IDENTIFY DEVICE command sent to ATAPI device
         * 4. REQUEST SENSE or READ LOG EXT command during error recovery
         */
        if (reset_flag ||
            ahci_portp->ahciport_device_type == SATA_DTYPE_NONE ||
            spkt && spkt->satapkt_cmd.satacmd_cmd_reg == SATAC_ID_DEVICE ||
            ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
                goto out;

        /*
         * Deliver READ LOG EXT to gather information about the error when
         * a COMRESET has not been performed as part of the error recovery
         * during NCQ command processing.
         */
        if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                failed_tags = ahci_get_rdlogext_data(ahci_ctlp,
                    ahci_portp, port);
                goto out;
        }

        /*
         * Deliver REQUEST SENSE for ATAPI command to gather information about
         * the error when a COMRESET has not been performed as part of the
         * error recovery.
         */
        if (spkt && ahci_portp->ahciport_device_type == SATA_DTYPE_ATAPI)
                ahci_get_rqsense_data(ahci_ctlp, ahci_portp, port, spkt);
out:
        AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
            "ahci_fatal_error_recovery_handler: port %d fatal error "
            "occurred slot_status = 0x%x, pending_tags = 0x%x, "
            "pending_ncq_tags = 0x%x failed_tags = 0x%x",
            port, slot_status, ahci_portp->ahciport_pending_tags,
            ahci_portp->ahciport_pending_ncq_tags, failed_tags);

        ahci_mop_commands(ahci_ctlp,
            ahci_portp,
            slot_status,
            failed_tags, /* failed tags */
            0, /* timeout tags */
            0, /* aborted tags */
            0); /* reset tags */
}

/*
 * Used to recovery a PMULT pmport fatal error under FIS-based switching.
 *      1. device specific.PxFBS.SDE=1
 *      2. Non Device specific.
 * Nothing will be done when Command-based switching is employed.
 *
 * Currently code is neither completed nor tested.
 */
static void
ahci_pmult_error_recovery_handler(ahci_ctl_t *ahci_ctlp,
    ahci_port_t *ahci_portp, uint8_t port, uint32_t intr_status)
{
#ifndef __lock_lint
        _NOTE(ARGUNUSED(intr_status))
#endif
        uint32_t        port_fbs_ctrl;
        int loop_count = 0;
        ahci_addr_t     addr;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        /* Nothing will be done under Command-based switching. */
        if (!(ahci_ctlp->ahcictl_cap & AHCI_CAP_PMULT_FBSS))
                return;

        port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));

        if (!(port_fbs_ctrl & AHCI_FBS_EN))
                /* FBS is not enabled. */
                return;

        /* Problem's getting complicated now. */
        /*
         * If FIS-based switching is used, we need to check
         * the PxFBS to see the error type.
         */
        port_fbs_ctrl = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port));

        /* Refer to spec(v1.2) 9.3.6.1 */
        if (port_fbs_ctrl & AHCI_FBS_SDE) {
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp,
                    "A Device Sepcific Error: port %d", port);
                /*
                 * Controller has paused commands for all other
                 * sub-devices until PxFBS.DEC is set.
                 */
                ahci_reject_all_abort_pkts(ahci_ctlp,
                    ahci_portp, 0);

                ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxFBS(ahci_ctlp, port),
                    port_fbs_ctrl | AHCI_FBS_DEC);

                /*
                 * Wait controller clear PxFBS.DEC,
                 * then we can continue.
                 */
                loop_count = 0;
                do {
                        port_fbs_ctrl = ddi_get32(ahci_ctlp->
                            ahcictl_ahci_acc_handle, (uint32_t *)
                            AHCI_PORT_PxFBS(ahci_ctlp, port));

                        if (loop_count++ > 1000)
                                /*
                                 * Esclate the error. Follow
                                 * non-device specific error
                                 * procedure.
                                 */
                                return;

                        drv_usecwait(AHCI_100US_USECS);
                } while (port_fbs_ctrl & AHCI_FBS_DEC);

                /*
                 * Issue a software reset to ensure drive is in
                 * a known state.
                 */
                (void) ahci_software_reset(ahci_ctlp,
                    ahci_portp, &addr);

        } else {

                /* Process Non-Device Specific Error. */
                /* This will be handled later on. */
                cmn_err(CE_NOTE, "!FBS is not supported now.");
        }
}
/*
 * Handle events - fatal error recovery
 */
static void
ahci_events_handler(void *args)
{
        ahci_event_arg_t *ahci_event_arg;
        ahci_ctl_t *ahci_ctlp;
        ahci_port_t *ahci_portp;
        ahci_addr_t *addrp;
        uint32_t event;
        int instance;

        ahci_event_arg = (ahci_event_arg_t *)args;

        ahci_ctlp = ahci_event_arg->ahciea_ctlp;
        ahci_portp = ahci_event_arg->ahciea_portp;
        addrp = ahci_event_arg->ahciea_addrp;
        event = ahci_event_arg->ahciea_event;
        instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);

        AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR|AHCIDBG_ERRS, ahci_ctlp,
            "ahci_events_handler enter: port %d intr_status = 0x%x",
            ahci_portp->ahciport_port_num, event);

        mutex_enter(&ahci_portp->ahciport_mutex);

        /*
         * ahci_intr_phyrdy_change() may have rendered it to
         * SATA_DTYPE_NONE.
         */
        if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
                AHCIDBG(AHCIDBG_ENTRY|AHCIDBG_INTR, ahci_ctlp,
                    "ahci_events_handler: port %d no device attached, "
                    "and just return without doing anything",
                    ahci_portp->ahciport_port_num);

                if (ahci_portp->ahciport_flags & AHCI_PORT_FLAG_ERRPRINT) {
                        ahci_portp->ahciport_flags &= ~AHCI_PORT_FLAG_ERRPRINT;
                        cmn_err(CE_WARN, "!ahci%d: error recovery for port %d "
                            "succeed", instance, ahci_portp->ahciport_port_num);
                }

                goto out;
        }

        if (event & (AHCI_INTR_STATUS_IFS |
            AHCI_INTR_STATUS_HBDS |
            AHCI_INTR_STATUS_HBFS |
            AHCI_INTR_STATUS_TFES))
                ahci_fatal_error_recovery_handler(ahci_ctlp, ahci_portp,
                    addrp, event);

out:
        mutex_exit(&ahci_portp->ahciport_mutex);
}

/*
 * ahci_watchdog_handler() and ahci_do_sync_start will call us if they
 * detect there are some commands which are timed out.
 */
static void
ahci_timeout_pkts(ahci_ctl_t *ahci_ctlp, ahci_port_t *ahci_portp,
    uint8_t port, uint32_t tmp_timeout_tags)
{
        uint32_t slot_status = 0;
        uint32_t finished_tags = 0;
        uint32_t timeout_tags = 0;

        AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_timeout_pkts enter: port %d", port);

        mutex_enter(&ahci_portp->ahciport_mutex);

        if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) ||
            RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp) ||
            ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
                /* Read PxCI to see which commands are still outstanding */
                slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCI(ahci_ctlp, port));
        } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                /* Read PxSACT to see which commands are still outstanding */
                slot_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
        }

#if AHCI_DEBUG
        /*
         * When AHCI_PORT_FLAG_RQSENSE or AHCI_PORT_FLAG_RDLOGEXT flag is
         * set, it means a fatal error happened after REQUEST SENSE command
         * or READ LOG EXT command is delivered to the HBA during the error
         * recovery process. At this time, the only outstanding command is
         * supposed to be REQUEST SENSE command or READ LOG EXT command.
         */
        if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp)) {
                AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
                    "ahci_timeout_pkts called while REQUEST SENSE "
                    "command or READ LOG EXT command for error recovery "
                    "timed out timeout_tags = 0x%x, slot_status = 0x%x, "
                    "pending_tags = 0x%x, pending_ncq_tags = 0x%x",
                    tmp_timeout_tags, slot_status,
                    ahci_portp->ahciport_pending_tags,
                    ahci_portp->ahciport_pending_ncq_tags);
                ASSERT(slot_status == 0x1);
        } else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
                AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT, ahci_ctlp,
                    "ahci_timeout_pkts called while executing R/W PMULT "
                    "command timeout_tags = 0x%x, slot_status = 0x%x",
                    tmp_timeout_tags, slot_status);
                ASSERT(slot_status == 0x1);
        }
#endif

        ahci_portp->ahciport_flags |= AHCI_PORT_FLAG_MOPPING;
        ahci_portp->ahciport_mop_in_progress++;

        (void) ahci_restart_port_wait_till_ready(ahci_ctlp, ahci_portp,
            port, AHCI_PORT_RESET, NULL);

        /*
         * Re-identify timeout tags because some previously checked commands
         * could already complete.
         */
        if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                finished_tags = ahci_portp->ahciport_pending_tags &
                    ~slot_status & AHCI_SLOT_MASK(ahci_ctlp);
                timeout_tags = tmp_timeout_tags & ~finished_tags;

                AHCIDBG(AHCIDBG_TIMEOUT, ahci_ctlp,
                    "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
                    "timeout_tags = 0x%x, port_cmd_issue = 0x%x, "
                    "pending_tags = 0x%x ",
                    port, finished_tags, timeout_tags,
                    slot_status, ahci_portp->ahciport_pending_tags);
        } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                finished_tags = ahci_portp->ahciport_pending_ncq_tags &
                    ~slot_status & AHCI_NCQ_SLOT_MASK(ahci_portp);
                timeout_tags = tmp_timeout_tags & ~finished_tags;

                AHCIDBG(AHCIDBG_TIMEOUT|AHCIDBG_NCQ, ahci_ctlp,
                    "ahci_timeout_pkts: port %d, finished_tags = 0x%x, "
                    "timeout_tags = 0x%x, port_sactive = 0x%x, "
                    "pending_ncq_tags = 0x%x ",
                    port, finished_tags, timeout_tags,
                    slot_status, ahci_portp->ahciport_pending_ncq_tags);
        } else if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
            RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
                timeout_tags = tmp_timeout_tags;
        }

        ahci_mop_commands(ahci_ctlp,
            ahci_portp,
            slot_status,
            0,                  /* failed tags */
            timeout_tags,       /* timeout tags */
            0,                  /* aborted tags */
            0);                 /* reset tags */

        mutex_exit(&ahci_portp->ahciport_mutex);
}

/*
 * Watchdog handler kicks in every 5 seconds to timeout any commands pending
 * for long time.
 */
static void
ahci_watchdog_handler(ahci_ctl_t *ahci_ctlp)
{
        ahci_port_t *ahci_portp;
        sata_pkt_t *spkt;
        uint32_t pending_tags;
        uint32_t timeout_tags;
        uint32_t port_cmd_status;
        uint32_t port_sactive;
        uint8_t port;
        int tmp_slot;
        int current_slot;
        uint32_t current_tags;
        int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);

        mutex_enter(&ahci_ctlp->ahcictl_mutex);

        AHCIDBG(AHCIDBG_ENTRY, ahci_ctlp,
            "ahci_watchdog_handler entered", NULL);

        current_slot = 0;
        current_tags = 0;
        for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
                if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
                        continue;
                }

                ahci_portp = ahci_ctlp->ahcictl_ports[port];

                mutex_enter(&ahci_portp->ahciport_mutex);
                if (ahci_portp->ahciport_device_type == SATA_DTYPE_NONE) {
                        mutex_exit(&ahci_portp->ahciport_mutex);
                        continue;
                }

                /* Skip the check for those ports in error recovery */
                if ((ahci_portp->ahciport_flags & AHCI_PORT_FLAG_MOPPING) &&
                    !(ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))) {
                        mutex_exit(&ahci_portp->ahciport_mutex);
                        continue;
                }

                pending_tags = 0;
                port_cmd_status = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_PORT_PxCMD(ahci_ctlp, port));

                if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp) ||
                    RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp)) {
                        current_slot = 0;
                        pending_tags = 0x1;
                } else if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                        current_slot =
                            (port_cmd_status & AHCI_CMD_STATUS_CCS) >>
                            AHCI_CMD_STATUS_CCS_SHIFT;
                        pending_tags = ahci_portp->ahciport_pending_tags;
                } else if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                        port_sactive = ddi_get32(
                            ahci_ctlp->ahcictl_ahci_acc_handle,
                            (uint32_t *)AHCI_PORT_PxSACT(ahci_ctlp, port));
                        current_tags = port_sactive &
                            ~port_cmd_status &
                            AHCI_NCQ_SLOT_MASK(ahci_portp);
                        pending_tags = ahci_portp->ahciport_pending_ncq_tags;
                }

                timeout_tags = 0;
                while (pending_tags) {
                        tmp_slot = ddi_ffs(pending_tags) - 1;
                        if (tmp_slot == -1) {
                                break;
                        }

                        if (ERR_RETRI_CMD_IN_PROGRESS(ahci_portp))
                                spkt = ahci_portp->ahciport_err_retri_pkt;
                        else if (RDWR_PMULT_CMD_IN_PROGRESS(ahci_portp))
                                spkt = ahci_portp->ahciport_rdwr_pmult_pkt;
                        else
                                spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];

                        if ((spkt != NULL) && spkt->satapkt_time &&
                            !(spkt->satapkt_op_mode & SATA_OPMODE_POLLING)) {
                                /*
                                 * If a packet has survived for more than it's
                                 * max life cycles, it is a candidate for time
                                 * out.
                                 */
                                ahci_portp->ahciport_slot_timeout[tmp_slot] -=
                                    ahci_watchdog_timeout;

                                if (ahci_portp->ahciport_slot_timeout[tmp_slot]
                                    > 0)
                                        goto next;

#if AHCI_DEBUG
                                if (NCQ_CMD_IN_PROGRESS(ahci_portp)) {
                                        AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
                                            ahci_ctlp, "watchdog: the current "
                                            "tags is 0x%x", current_tags);
                                } else {
                                        AHCIDBG(AHCIDBG_ERRS|AHCIDBG_TIMEOUT,
                                            ahci_ctlp, "watchdog: the current "
                                            "slot is %d", current_slot);
                                }
#endif

                                /*
                                 * We need to check whether the HBA has
                                 * begun to execute the command, if not,
                                 * then re-set the timer of the command.
                                 */
                                if (NON_NCQ_CMD_IN_PROGRESS(ahci_portp) &&
                                    (tmp_slot != current_slot) ||
                                    NCQ_CMD_IN_PROGRESS(ahci_portp) &&
                                    ((0x1 << tmp_slot) & current_tags)) {
                                        ahci_portp->ahciport_slot_timeout \
                                            [tmp_slot] = spkt->satapkt_time;
                                } else {
                                        timeout_tags |= (0x1 << tmp_slot);
                                        cmn_err(CE_WARN, "!ahci%d: watchdog "
                                            "port %d satapkt 0x%p timed out\n",
                                            instance, port, (void *)spkt);
                                }
                        }
next:
                        CLEAR_BIT(pending_tags, tmp_slot);
                }

                if (timeout_tags) {
                        mutex_exit(&ahci_portp->ahciport_mutex);
                        mutex_exit(&ahci_ctlp->ahcictl_mutex);
                        ahci_timeout_pkts(ahci_ctlp, ahci_portp,
                            port, timeout_tags);
                        mutex_enter(&ahci_ctlp->ahcictl_mutex);
                        mutex_enter(&ahci_portp->ahciport_mutex);
                }

                mutex_exit(&ahci_portp->ahciport_mutex);
        }

        /* Re-install the watchdog timeout handler */
        if (ahci_ctlp->ahcictl_timeout_id != 0) {
                ahci_ctlp->ahcictl_timeout_id =
                    timeout((void (*)(void *))ahci_watchdog_handler,
                    (caddr_t)ahci_ctlp, ahci_watchdog_tick);
        }

        mutex_exit(&ahci_ctlp->ahcictl_mutex);
}

/*
 * Fill the error context into sata_cmd for non-queued command error.
 */
static void
ahci_copy_err_cnxt(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
{
        scmd->satacmd_status_reg = GET_RFIS_STATUS(rfisp);
        scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
        scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
        scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
        scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
        scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
        scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);

        if (scmd->satacmd_addr_type == ATA_ADDR_LBA48) {
                scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
                scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
                scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
                scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
        }
}

/*
 * Fill the ncq error page into sata_cmd for queued command error.
 */
static void
ahci_copy_ncq_err_page(sata_cmd_t *scmd,
    struct sata_ncq_error_recovery_page *ncq_err_page)
{
        scmd->satacmd_sec_count_msb = ncq_err_page->ncq_sector_count_ext;
        scmd->satacmd_sec_count_lsb = ncq_err_page->ncq_sector_count;
        scmd->satacmd_lba_low_msb = ncq_err_page->ncq_sector_number_ext;
        scmd->satacmd_lba_low_lsb = ncq_err_page->ncq_sector_number;
        scmd->satacmd_lba_mid_msb = ncq_err_page->ncq_cyl_low_ext;
        scmd->satacmd_lba_mid_lsb = ncq_err_page->ncq_cyl_low;
        scmd->satacmd_lba_high_msb = ncq_err_page->ncq_cyl_high_ext;
        scmd->satacmd_lba_high_lsb = ncq_err_page->ncq_cyl_high;
        scmd->satacmd_device_reg = ncq_err_page->ncq_dev_head;
        scmd->satacmd_status_reg = ncq_err_page->ncq_status;
        scmd->satacmd_error_reg = ncq_err_page->ncq_error;
}

/*
 * Put the respective register value to sata_cmd_t for satacmd_flags.
 */
static void
ahci_copy_out_regs(sata_cmd_t *scmd, ahci_fis_d2h_register_t *rfisp)
{
        if (scmd->satacmd_flags.sata_copy_out_sec_count_msb)
                scmd->satacmd_sec_count_msb = GET_RFIS_SECTOR_COUNT_EXP(rfisp);
        if (scmd->satacmd_flags.sata_copy_out_lba_low_msb)
                scmd->satacmd_lba_low_msb = GET_RFIS_CYL_LOW_EXP(rfisp);
        if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb)
                scmd->satacmd_lba_mid_msb = GET_RFIS_CYL_MID_EXP(rfisp);
        if (scmd->satacmd_flags.sata_copy_out_lba_high_msb)
                scmd->satacmd_lba_high_msb = GET_RFIS_CYL_HI_EXP(rfisp);
        if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb)
                scmd->satacmd_sec_count_lsb = GET_RFIS_SECTOR_COUNT(rfisp);
        if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb)
                scmd->satacmd_lba_low_lsb = GET_RFIS_CYL_LOW(rfisp);
        if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb)
                scmd->satacmd_lba_mid_lsb = GET_RFIS_CYL_MID(rfisp);
        if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb)
                scmd->satacmd_lba_high_lsb = GET_RFIS_CYL_HI(rfisp);
        if (scmd->satacmd_flags.sata_copy_out_device_reg)
                scmd->satacmd_device_reg = GET_RFIS_DEV_HEAD(rfisp);
        if (scmd->satacmd_flags.sata_copy_out_error_reg)
                scmd->satacmd_error_reg = GET_RFIS_ERROR(rfisp);
}

static void
ahci_log_fatal_error_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
    uint32_t intr_status)
{
        int instance = ddi_get_instance(ahci_ctlp->ahcictl_dip);

        if (intr_status & AHCI_INTR_STATUS_IFS)
                cmn_err(CE_WARN, "!ahci%d: ahci port %d has interface fatal "
                    "error", instance, port);

        if (intr_status & AHCI_INTR_STATUS_HBDS)
                cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus data error",
                    instance, port);

        if (intr_status & AHCI_INTR_STATUS_HBFS)
                cmn_err(CE_WARN, "!ahci%d: ahci port %d has bus fatal error",
                    instance, port);

        if (intr_status & AHCI_INTR_STATUS_TFES)
                cmn_err(CE_WARN, "!ahci%d: ahci port %d has task file error",
                    instance, port);

        cmn_err(CE_WARN, "!ahci%d: ahci port %d is trying to do error "
            "recovery", instance, port);
}

static void
ahci_dump_commands(ahci_ctl_t *ahci_ctlp, uint8_t port,
    uint32_t slot_tags)
{
        ahci_port_t *ahci_portp;
        int tmp_slot;
        sata_pkt_t *spkt;
        sata_cmd_t cmd;

        ahci_portp = ahci_ctlp->ahcictl_ports[port];
        ASSERT(ahci_portp != NULL);

        while (slot_tags) {
                tmp_slot = ddi_ffs(slot_tags) - 1;
                if (tmp_slot == -1) {
                        break;
                }

                spkt = ahci_portp->ahciport_slot_pkts[tmp_slot];
                if (spkt != NULL) {
                        cmd = spkt->satapkt_cmd;

                        cmn_err(CE_WARN, "!satapkt 0x%p: cmd_reg = 0x%x "
                            "features_reg = 0x%x sec_count_msb = 0x%x "
                            "lba_low_msb = 0x%x lba_mid_msb = 0x%x "
                            "lba_high_msb = 0x%x sec_count_lsb = 0x%x "
                            "lba_low_lsb = 0x%x lba_mid_lsb = 0x%x "
                            "lba_high_lsb = 0x%x device_reg = 0x%x "
                            "addr_type = 0x%x cmd_flags = 0x%x", (void *)spkt,
                            cmd.satacmd_cmd_reg, cmd.satacmd_features_reg,
                            cmd.satacmd_sec_count_msb, cmd.satacmd_lba_low_msb,
                            cmd.satacmd_lba_mid_msb, cmd.satacmd_lba_high_msb,
                            cmd.satacmd_sec_count_lsb, cmd.satacmd_lba_low_lsb,
                            cmd.satacmd_lba_mid_lsb, cmd.satacmd_lba_high_lsb,
                            cmd.satacmd_device_reg, cmd.satacmd_addr_type,
                            *((uint32_t *)&(cmd.satacmd_flags)));
                }

                CLEAR_BIT(slot_tags, tmp_slot);
        }
}

/*
 * Dump the serror message to the log.
 */
static void
ahci_log_serror_message(ahci_ctl_t *ahci_ctlp, uint8_t port,
    uint32_t port_serror, int debug_only)
{
        static char err_buf[512];
        static char err_msg_header[16];
        char *err_msg = err_buf;

        *err_buf = '\0';
        *err_msg_header = '\0';

        if (port_serror & SERROR_DATA_ERR_FIXED) {
                err_msg = strcat(err_msg,
                    "\tRecovered Data Integrity Error (I)\n");
        }

        if (port_serror & SERROR_COMM_ERR_FIXED) {
                err_msg = strcat(err_msg,
                    "\tRecovered Communication Error (M)\n");
        }

        if (port_serror & SERROR_DATA_ERR) {
                err_msg = strcat(err_msg,
                    "\tTransient Data Integrity Error (T)\n");
        }

        if (port_serror & SERROR_PERSISTENT_ERR) {
                err_msg = strcat(err_msg,
                    "\tPersistent Communication or Data Integrity Error (C)\n");
        }

        if (port_serror & SERROR_PROTOCOL_ERR) {
                err_msg = strcat(err_msg, "\tProtocol Error (P)\n");
        }

        if (port_serror & SERROR_INT_ERR) {
                err_msg = strcat(err_msg, "\tInternal Error (E)\n");
        }

        if (port_serror & SERROR_PHY_RDY_CHG) {
                err_msg = strcat(err_msg, "\tPhyRdy Change (N)\n");
        }

        if (port_serror & SERROR_PHY_INT_ERR) {
                err_msg = strcat(err_msg, "\tPhy Internal Error (I)\n");
        }

        if (port_serror & SERROR_COMM_WAKE) {
                err_msg = strcat(err_msg, "\tComm Wake (W)\n");
        }

        if (port_serror & SERROR_10B_TO_8B_ERR) {
                err_msg = strcat(err_msg, "\t10B to 8B Decode Error (B)\n");
        }

        if (port_serror & SERROR_DISPARITY_ERR) {
                err_msg = strcat(err_msg, "\tDisparity Error (D)\n");
        }

        if (port_serror & SERROR_CRC_ERR) {
                err_msg = strcat(err_msg, "\tCRC Error (C)\n");
        }

        if (port_serror & SERROR_HANDSHAKE_ERR) {
                err_msg = strcat(err_msg, "\tHandshake Error (H)\n");
        }

        if (port_serror & SERROR_LINK_SEQ_ERR) {
                err_msg = strcat(err_msg, "\tLink Sequence Error (S)\n");
        }

        if (port_serror & SERROR_TRANS_ERR) {
                err_msg = strcat(err_msg,
                    "\tTransport state transition error (T)\n");
        }

        if (port_serror & SERROR_FIS_TYPE) {
                err_msg = strcat(err_msg, "\tUnknown FIS Type (F)\n");
        }

        if (port_serror & SERROR_EXCHANGED_ERR) {
                err_msg = strcat(err_msg, "\tExchanged (X)\n");
        }

        if (*err_msg == '\0')
                return;

        if (debug_only) {
                (void) sprintf(err_msg_header, "port %d", port);
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg_header, NULL);
                AHCIDBG(AHCIDBG_ERRS, ahci_ctlp, err_msg, NULL);
        } else if (ahci_ctlp) {
                cmn_err(CE_WARN, "!ahci%d: %s %s",
                    ddi_get_instance(ahci_ctlp->ahcictl_dip),
                    err_msg_header, err_msg);

                /* sata trace debug */
                sata_trace_debug(ahci_ctlp->ahcictl_dip,
                    "ahci%d: %s %s", ddi_get_instance(ahci_ctlp->ahcictl_dip),
                    err_msg_header, err_msg);
        } else {
                cmn_err(CE_WARN, "!ahci: %s %s", err_msg_header, err_msg);

                /* sata trace debug */
                sata_trace_debug(NULL, "ahci: %s %s", err_msg_header, err_msg);
        }
}

/*
 * Translate the sata_address_t type into the ahci_addr_t type.
 * sata_device.satadev_addr structure is used as source.
 */
static void
ahci_get_ahci_addr(ahci_ctl_t *ahci_ctlp, sata_device_t *sd,
    ahci_addr_t *ahci_addrp)
{
        sata_address_t *sata_addrp = &sd->satadev_addr;
        ahci_addrp->aa_port =
            ahci_ctlp->ahcictl_cport_to_port[sata_addrp->cport];
        ahci_addrp->aa_pmport = sata_addrp->pmport;

        switch (sata_addrp->qual) {
        case SATA_ADDR_DCPORT:
        case SATA_ADDR_CPORT:
                ahci_addrp->aa_qual = AHCI_ADDR_PORT;
                break;
        case SATA_ADDR_PMULT:
        case SATA_ADDR_PMULT_SPEC:
                ahci_addrp->aa_qual = AHCI_ADDR_PMULT;
                break;
        case SATA_ADDR_DPMPORT:
        case SATA_ADDR_PMPORT:
                ahci_addrp->aa_qual = AHCI_ADDR_PMPORT;
                break;
        case SATA_ADDR_NULL:
        default:
                /* something went wrong */
                ahci_addrp->aa_qual = AHCI_ADDR_NULL;
                break;
        }
}

/*
 * This routine is to calculate the total number of ports implemented
 * by the HBA.
 */
static int
ahci_get_num_implemented_ports(uint32_t ports_implemented)
{
        uint8_t i;
        int num = 0;

        for (i = 0; i < AHCI_MAX_PORTS; i++) {
                if (((uint32_t)0x1 << i) & ports_implemented)
                        num++;
        }

        return (num);
}

#if AHCI_DEBUG
static void
ahci_log(ahci_ctl_t *ahci_ctlp, uint_t level, char *fmt, ...)
{
        static char name[16];
        va_list ap;

        mutex_enter(&ahci_log_mutex);

        va_start(ap, fmt);
        if (ahci_ctlp) {
                (void) sprintf(name, "ahci%d: ",
                    ddi_get_instance(ahci_ctlp->ahcictl_dip));
        } else {
                (void) sprintf(name, "ahci: ");
        }

        (void) vsprintf(ahci_log_buf, fmt, ap);
        va_end(ap);

        cmn_err(level, "%s%s", name, ahci_log_buf);

        mutex_exit(&ahci_log_mutex);
}
#endif

/*
 * quiesce(9E) entry point.
 *
 * This function is called when the system is single-threaded at high
 * PIL with preemption disabled. Therefore, this function must not be
 * blocked. Because no taskqs are running, there is no need for us to
 * take any action for enclosure services which are running in the
 * taskq context, especially as no interrupts are generated by it nor
 * are any messages expected to come in.
 *
 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
 * DDI_FAILURE indicates an error condition and should almost never happen.
 */
static int
ahci_quiesce(dev_info_t *dip)
{
        ahci_ctl_t *ahci_ctlp;
        ahci_port_t *ahci_portp;
        int instance, port;

        instance = ddi_get_instance(dip);
        ahci_ctlp = ddi_get_soft_state(ahci_statep, instance);

        if (ahci_ctlp == NULL)
                return (DDI_FAILURE);

#if AHCI_DEBUG
        ahci_debug_flags = 0;
#endif

        ahci_ctlp->ahcictl_flags |= AHCI_QUIESCE;

        /* disable all the interrupts. */
        ahci_disable_all_intrs(ahci_ctlp);

        for (port = 0; port < ahci_ctlp->ahcictl_num_ports; port++) {
                if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, port)) {
                        continue;
                }

                ahci_portp = ahci_ctlp->ahcictl_ports[port];

                /*
                 * Stop the port by clearing PxCMD.ST
                 *
                 * Here we must disable the port interrupt because
                 * ahci_disable_all_intrs only clear GHC.IE, and IS
                 * register will be still set if PxIE is enabled.
                 * When ahci shares one IRQ with other drivers, the
                 * intr handler may claim the intr mistakenly.
                 */
                ahci_disable_port_intrs(ahci_ctlp, port);
                (void) ahci_put_port_into_notrunning_state(ahci_ctlp,
                    ahci_portp, port);
        }

        ahci_ctlp->ahcictl_flags &= ~AHCI_QUIESCE;

        return (DDI_SUCCESS);
}

/*
 * The function will add a sata packet to the done queue.
 */
static void
ahci_add_doneq(ahci_port_t *ahci_portp, sata_pkt_t *satapkt, int reason)
{
        ASSERT(satapkt != NULL);
        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        /* set the reason for all packets */
        satapkt->satapkt_reason = reason;
        satapkt->satapkt_hba_driver_private = NULL;

        if (! (satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
            satapkt->satapkt_comp) {
                /*
                 * only add to queue when mode is not synch and there is
                 * completion callback
                 */
                *ahci_portp->ahciport_doneqtail = satapkt;
                ahci_portp->ahciport_doneqtail =
                    (sata_pkt_t **)&(satapkt->satapkt_hba_driver_private);
                ahci_portp->ahciport_doneq_len++;

        } else if ((satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&
            ! (satapkt->satapkt_op_mode & SATA_OPMODE_POLLING))
                /*
                 * for sync/non-poll mode, just call cv_broadcast
                 */
                cv_broadcast(&ahci_portp->ahciport_cv);
}

/*
 * The function will call completion callback of sata packet on the
 * completed queue
 */
static void
ahci_flush_doneq(ahci_port_t *ahci_portp)
{
        sata_pkt_t *satapkt, *next;

        ASSERT(MUTEX_HELD(&ahci_portp->ahciport_mutex));

        if (ahci_portp->ahciport_doneq) {
                satapkt = ahci_portp->ahciport_doneq;

                ahci_portp->ahciport_doneq = NULL;
                ahci_portp->ahciport_doneqtail = &ahci_portp->ahciport_doneq;
                ahci_portp->ahciport_doneq_len = 0;

                mutex_exit(&ahci_portp->ahciport_mutex);

                while (satapkt != NULL) {
                        next = satapkt->satapkt_hba_driver_private;
                        satapkt->satapkt_hba_driver_private = NULL;

                        /* Call the callback */
                        (*satapkt->satapkt_comp)(satapkt);

                        satapkt = next;
                }

                mutex_enter(&ahci_portp->ahciport_mutex);
        }
}

/*
 * Sets the state for the specified port on the controller to desired state.
 * This must be run in the context of the enclosure taskq which ensures that
 * only one event is outstanding at any time.
 */
static boolean_t
ahci_em_set_led(ahci_ctl_t *ahci_ctlp, uint8_t port, ahci_em_led_state_t desire)
{
        ahci_em_led_msg_t msg;
        ahci_em_msg_hdr_t hdr;
        uint32_t msgval, hdrval;
        uint_t i, max_delay = ahci_em_tx_delay_count;

        msg.alm_hba = port;
        msg.alm_pminfo = 0;
        msg.alm_value = 0;

        if (desire & AHCI_EM_LED_IDENT_ENABLE) {
                msg.alm_value |= AHCI_LED_ON << AHCI_LED_IDENT_OFF;
        }

        if (desire & AHCI_EM_LED_FAULT_ENABLE) {
                msg.alm_value |= AHCI_LED_ON << AHCI_LED_FAULT_OFF;
        }

        if ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) == 0 &&
            (desire & AHCI_EM_LED_ACTIVITY_DISABLE) == 0) {
                msg.alm_value |= AHCI_LED_ON << AHCI_LED_ACTIVITY_OFF;
        }

        hdr.aemh_rsvd = 0;
        hdr.aemh_mlen = sizeof (ahci_em_led_msg_t);
        hdr.aemh_dlen = 0;
        hdr.aemh_mtype = AHCI_EM_MSG_TYPE_LED;

        bcopy(&msg, &msgval, sizeof (msgval));
        bcopy(&hdr, &hdrval, sizeof (hdrval));

        /*
         * First, make sure we can transmit. We should not have been placed in a
         * situation where an outstanding transmission is going on.
         */
        for (i = 0; i < max_delay; i++) {
                uint32_t val;

                val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
                if ((val & AHCI_HBA_EM_CTL_CTL_TM) == 0)
                        break;

                delay(drv_usectohz(ahci_em_tx_delay_ms * 1000));
        }

        if (i == max_delay)
                return (B_FALSE);

        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)ahci_ctlp->ahcictl_em_tx_off, hdrval);
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)(ahci_ctlp->ahcictl_em_tx_off + 4), msgval);
        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp), AHCI_HBA_EM_CTL_CTL_TM);

        for (i = 0; i < max_delay; i++) {
                uint32_t val;

                val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
                if ((val & AHCI_HBA_EM_CTL_CTL_TM) == 0)
                        break;

                delay(drv_usectohz(ahci_em_tx_delay_ms * 1000));
        }

        if (i == max_delay)
                return (B_FALSE);

        return (B_TRUE);
}

typedef struct ahci_em_led_task_arg {
        ahci_ctl_t              *aelta_ctl;
        uint8_t                 aelta_port;
        uint_t                  aelta_op;
        ahci_em_led_state_t     aelta_state;
        uint_t                  aelta_ret;
        kcondvar_t              aelta_cv;
        uint_t                  aelta_ref;
} ahci_em_led_task_arg_t;

static void
ahci_em_led_task_free(ahci_em_led_task_arg_t *task)
{
        ASSERT3U(task->aelta_ref, ==, 0);
        cv_destroy(&task->aelta_cv);
        kmem_free(task, sizeof (*task));
}

static void
ahci_em_led_task(void *arg)
{
        boolean_t ret, cleanup = B_FALSE;
        ahci_em_led_task_arg_t *led = arg;
        ahci_em_led_state_t state;

        mutex_enter(&led->aelta_ctl->ahcictl_mutex);
        if (led->aelta_ctl->ahcictl_em_flags != AHCI_EM_USABLE) {
                led->aelta_ret = EIO;
                mutex_exit(&led->aelta_ctl->ahcictl_mutex);
                return;
        }

        state = led->aelta_ctl->ahcictl_em_state[led->aelta_port];
        mutex_exit(&led->aelta_ctl->ahcictl_mutex);

        switch (led->aelta_op) {
        case AHCI_EM_IOC_SET_OP_ADD:
                state |= led->aelta_state;
                break;
        case AHCI_EM_IOC_SET_OP_REM:
                state &= ~led->aelta_state;
                break;
        case AHCI_EM_IOC_SET_OP_SET:
                state = led->aelta_state;
                break;
        default:
                led->aelta_ret = ENOTSUP;
                return;
        }

        ret = ahci_em_set_led(led->aelta_ctl, led->aelta_port, state);

        mutex_enter(&led->aelta_ctl->ahcictl_mutex);
        if (ret) {
                led->aelta_ctl->ahcictl_em_state[led->aelta_port] = state;
                led->aelta_ret = 0;
        } else {
                led->aelta_ret = EIO;
                led->aelta_ctl->ahcictl_em_flags |= AHCI_EM_TIMEOUT;
        }
        led->aelta_ref--;
        if (led->aelta_ref > 0) {
                cv_signal(&led->aelta_cv);
        } else {
                cleanup = B_TRUE;
        }
        mutex_exit(&led->aelta_ctl->ahcictl_mutex);

        if (cleanup) {
                ahci_em_led_task_free(led);
        }
}

static void
ahci_em_reset(void *arg)
{
        uint_t i, max_delay = ahci_em_reset_delay_count;
        ahci_ctl_t *ahci_ctlp = arg;

        /*
         * We've been asked to reset the device. The caller should have set the
         * resetting flag. Make sure that we don't have a request to quiesce.
         */
        mutex_enter(&ahci_ctlp->ahcictl_mutex);
        ASSERT(ahci_ctlp->ahcictl_em_flags & AHCI_EM_RESETTING);
        if (ahci_ctlp->ahcictl_em_flags & AHCI_EM_QUIESCE) {
                ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
                mutex_exit(&ahci_ctlp->ahcictl_mutex);
                return;
        }
        mutex_exit(&ahci_ctlp->ahcictl_mutex);

        ddi_put32(ahci_ctlp->ahcictl_ahci_acc_handle,
            (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp), AHCI_HBA_EM_CTL_CTL_RST);
        for (i = 0; i < max_delay; i++) {
                uint32_t val;

                val = ddi_get32(ahci_ctlp->ahcictl_ahci_acc_handle,
                    (uint32_t *)AHCI_GLOBAL_EM_CTL(ahci_ctlp));
                if ((val & AHCI_HBA_EM_CTL_CTL_RST) == 0)
                        break;

                delay(drv_usectohz(ahci_em_reset_delay_ms * 1000));
        }

        if (i == max_delay) {
                mutex_enter(&ahci_ctlp->ahcictl_mutex);
                ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
                ahci_ctlp->ahcictl_em_flags |= AHCI_EM_TIMEOUT;
                mutex_exit(&ahci_ctlp->ahcictl_mutex);
                cmn_err(CE_WARN, "!ahci%d: enclosure timed out resetting",
                    ddi_get_instance(ahci_ctlp->ahcictl_dip));
                return;
        }

        for (i = 0; i < ahci_ctlp->ahcictl_num_ports; i++) {

                if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, i))
                        continue;

                /*
                 * Try to flush all the LEDs as part of reset. If it fails,
                 * drive on.
                 */
                if (!ahci_em_set_led(ahci_ctlp, i,
                    ahci_ctlp->ahcictl_em_state[i])) {
                        mutex_enter(&ahci_ctlp->ahcictl_mutex);
                        ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
                        ahci_ctlp->ahcictl_em_flags |= AHCI_EM_TIMEOUT;
                        mutex_exit(&ahci_ctlp->ahcictl_mutex);
                        cmn_err(CE_WARN, "!ahci%d: enclosure timed out "
                            "setting port %u",
                            ddi_get_instance(ahci_ctlp->ahcictl_dip), i);
                        return;
                }
        }

        mutex_enter(&ahci_ctlp->ahcictl_mutex);
        ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_RESETTING;
        ahci_ctlp->ahcictl_em_flags |= AHCI_EM_READY;
        mutex_exit(&ahci_ctlp->ahcictl_mutex);
}

static boolean_t
ahci_em_init(ahci_ctl_t *ahci_ctlp)
{
        char name[128];

        /*
         * First make sure we actually have enclosure services and if so, that
         * we have the hardware support that we care about for this.
         */
        if (ahci_ctlp->ahcictl_em_loc == 0 ||
            (ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_SUPP_LED) == 0)
                return (B_TRUE);

        /*
         * Next, make sure that the buffer is large enough for us. We need two
         * dwords or 8 bytes. The location register is stored in dwords.
         */
        if ((ahci_ctlp->ahcictl_em_loc & AHCI_HBA_EM_LOC_SZ_MASK) <
            AHCI_EM_BUFFER_MIN) {
                return (B_TRUE);
        }

        ahci_ctlp->ahcictl_em_tx_off = ((ahci_ctlp->ahcictl_em_loc &
            AHCI_HBA_EM_LOC_OFST_MASK) >> AHCI_HBA_EM_LOC_OFST_SHIFT) * 4;
        ahci_ctlp->ahcictl_em_tx_off += ahci_ctlp->ahcictl_ahci_addr;

        bzero(ahci_ctlp->ahcictl_em_state,
            sizeof (ahci_ctlp->ahcictl_em_state));

        (void) snprintf(name, sizeof (name), "ahcti_em_taskq%d",
            ddi_get_instance(ahci_ctlp->ahcictl_dip));
        if ((ahci_ctlp->ahcictl_em_taskq =
            ddi_taskq_create(ahci_ctlp->ahcictl_dip, name, 1,
            TASKQ_DEFAULTPRI, 0)) == NULL) {
                cmn_err(CE_WARN, "!ahci%d: ddi_tasq_create failed for em "
                    "services", ddi_get_instance(ahci_ctlp->ahcictl_dip));
                return (B_FALSE);
        }

        mutex_enter(&ahci_ctlp->ahcictl_mutex);
        ahci_ctlp->ahcictl_em_flags |= AHCI_EM_PRESENT | AHCI_EM_RESETTING;
        mutex_exit(&ahci_ctlp->ahcictl_mutex);
        (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq, ahci_em_reset,
            ahci_ctlp, DDI_SLEEP);

        return (B_TRUE);
}

static int
ahci_em_ioctl_get(ahci_ctl_t *ahci_ctlp, intptr_t arg)
{
        int i;
        ahci_ioc_em_get_t get;

        if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
                return (ENOTSUP);
        }

        bzero(&get, sizeof (get));
        get.aiemg_nports = ahci_ctlp->ahcictl_ports_implemented;
        if ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) == 0) {
                get.aiemg_flags |= AHCI_EM_FLAG_CONTROL_ACTIVITY;
        }

        mutex_enter(&ahci_ctlp->ahcictl_mutex);
        for (i = 0; i < ahci_ctlp->ahcictl_num_ports; i++) {
                if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, i)) {
                        continue;
                }
                get.aiemg_status[i] = ahci_ctlp->ahcictl_em_state[i];
        }
        mutex_exit(&ahci_ctlp->ahcictl_mutex);

        if (ddi_copyout(&get, (void *)arg, sizeof (get), 0) != 0)
                return (EFAULT);

        return (0);
}

static int
ahci_em_ioctl_set(ahci_ctl_t *ahci_ctlp, intptr_t arg)
{
        int ret;
        ahci_ioc_em_set_t set;
        ahci_em_led_task_arg_t *task;
        boolean_t signal, cleanup;

        if (ddi_copyin((void *)arg, &set, sizeof (set), 0) != 0)
                return (EFAULT);

        if (set.aiems_port > ahci_ctlp->ahcictl_num_ports)
                return (EINVAL);

        if (!AHCI_PORT_IMPLEMENTED(ahci_ctlp, set.aiems_port)) {
                return (EINVAL);
        }

        if ((set.aiems_leds & ~(AHCI_EM_LED_IDENT_ENABLE |
            AHCI_EM_LED_FAULT_ENABLE |
            AHCI_EM_LED_ACTIVITY_DISABLE)) != 0) {
                return (EINVAL);
        }

        switch (set.aiems_op) {
        case AHCI_EM_IOC_SET_OP_ADD:
        case AHCI_EM_IOC_SET_OP_REM:
        case AHCI_EM_IOC_SET_OP_SET:
                break;
        default:
                return (EINVAL);
        }

        if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
                return (ENOTSUP);
        }

        if ((set.aiems_leds & AHCI_EM_LED_ACTIVITY_DISABLE) != 0 &&
            ((ahci_ctlp->ahcictl_em_ctl & AHCI_HBA_EM_CTL_ATTR_ALHD) != 0)) {
                return (ENOTSUP);
        }

        task = kmem_alloc(sizeof (*task), KM_NOSLEEP_LAZY);
        if (task == NULL) {
                return (ENOMEM);
        }

        task->aelta_ctl = ahci_ctlp;
        task->aelta_port = (uint8_t)set.aiems_port;
        task->aelta_op = set.aiems_op;
        task->aelta_state = set.aiems_leds;

        cv_init(&task->aelta_cv, NULL, CV_DRIVER, NULL);

        /*
         * Initialize the reference count to two. One for us and one for the
         * taskq. This will be used in case we get canceled.
         */
        task->aelta_ref = 2;

        /*
         * Once dispatched, the task state is protected by our global mutex.
         */
        (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq,
            ahci_em_led_task, task, DDI_SLEEP);

        signal = B_FALSE;
        mutex_enter(&ahci_ctlp->ahcictl_mutex);
        while (task->aelta_ref > 1) {
                if (cv_wait_sig(&task->aelta_cv, &ahci_ctlp->ahcictl_mutex) ==
                    0) {
                        signal = B_TRUE;
                        break;
                }
        }

        /*
         * Remove our reference count. If we were woken up because of a signal
         * then the taskq may still be dispatched. In which case we shouldn't
         * free this memory until it is done. In that case, the taskq will take
         * care of it.
         */
        task->aelta_ref--;
        cleanup = (task->aelta_ref == 0);
        if (signal) {
                ret = EINTR;
        } else {
                ret = task->aelta_ret;
        }
        mutex_exit(&ahci_ctlp->ahcictl_mutex);

        if (cleanup) {
                ahci_em_led_task_free(task);
        }

        return (ret);
}

static int
ahci_em_ioctl(dev_info_t *dip, int cmd, intptr_t arg)
{
        int inst;
        ahci_ctl_t *ahci_ctlp;

        inst = ddi_get_instance(dip);
        if ((ahci_ctlp = ddi_get_soft_state(ahci_statep, inst)) == NULL) {
                return (ENXIO);
        }

        switch (cmd) {
        case AHCI_EM_IOC_GET:
                return (ahci_em_ioctl_get(ahci_ctlp, arg));
        case AHCI_EM_IOC_SET:
                return (ahci_em_ioctl_set(ahci_ctlp, arg));
        default:
                return (ENOTTY);
        }

}

static void
ahci_em_quiesce(ahci_ctl_t *ahci_ctlp)
{
        mutex_enter(&ahci_ctlp->ahcictl_mutex);
        if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
                mutex_exit(&ahci_ctlp->ahcictl_mutex);
                return;
        }
        ahci_ctlp->ahcictl_em_flags |= AHCI_EM_QUIESCE;
        mutex_exit(&ahci_ctlp->ahcictl_mutex);

        ddi_taskq_wait(ahci_ctlp->ahcictl_em_taskq);
}

static void
ahci_em_suspend(ahci_ctl_t *ahci_ctlp)
{
        ahci_em_quiesce(ahci_ctlp);

        mutex_enter(&ahci_ctlp->ahcictl_mutex);
        ahci_ctlp->ahcictl_em_flags &= ~AHCI_EM_READY;
        mutex_exit(&ahci_ctlp->ahcictl_mutex);
}

static void
ahci_em_resume(ahci_ctl_t *ahci_ctlp)
{
        mutex_enter(&ahci_ctlp->ahcictl_mutex);
        if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
                mutex_exit(&ahci_ctlp->ahcictl_mutex);
                return;
        }
        ahci_ctlp->ahcictl_em_flags |= AHCI_EM_RESETTING;
        mutex_exit(&ahci_ctlp->ahcictl_mutex);

        (void) ddi_taskq_dispatch(ahci_ctlp->ahcictl_em_taskq, ahci_em_reset,
            ahci_ctlp, DDI_SLEEP);
}

static void
ahci_em_fini(ahci_ctl_t *ahci_ctlp)
{
        if ((ahci_ctlp->ahcictl_em_flags & AHCI_EM_PRESENT) == 0) {
                return;
        }

        ahci_em_quiesce(ahci_ctlp);
        ddi_taskq_destroy(ahci_ctlp->ahcictl_em_taskq);
        ahci_ctlp->ahcictl_em_taskq = NULL;
}