/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2006, 2010, Oracle and/or its affiliates. All rights reserved.
 */


/*
 * SiliconImage 3124/3132/3531 sata controller driver
 */

/*
 *
 *
 *                      Few Design notes
 *
 *
 * I. General notes
 *
 * Even though the driver is named as si3124, it is actually meant to
 * work with SiI3124, SiI3132 and SiI3531 controllers.
 *
 * The current file si3124.c is the main driver code. The si3124reg.h
 * holds the register definitions from SiI 3124/3132/3531 data sheets. The
 * si3124var.h holds the driver specific definitions which are not
 * directly derived from data sheets.
 *
 *
 * II. Data structures
 *
 * si_ctl_state_t: This holds the driver private information for each
 *      controller instance. Each of the sata ports within a single
 *      controller are represented by si_port_state_t. The
 *      sictl_global_acc_handle and sictl_global_address map the
 *      controller-wide global register space and are derived from pci
 *      BAR 0. The sictl_port_acc_handle and sictl_port_addr map the
 *      per-port register space and are derived from pci BAR 1.
 *
 * si_port_state_t: This holds the per port information. The siport_mutex
 *      holds the per port mutex. The siport_pending_tags is the bit mask of
 *      commands posted to controller. The siport_slot_pkts[] holds the
 *      pending sata packets. The siport_port_type holds the device type
 *      connected directly to the port while the siport_portmult_state
 *      holds the similar information for the devices behind a port
 *      multiplier.
 *
 * si_prb_t: This contains the PRB being posted to the controller.
 *      The two SGE entries contained within si_prb_t itself are not
 *      really used to hold any scatter gather entries. The scatter gather
 *      list is maintained external to PRB and is linked from one
 *      of the contained SGEs inside the PRB. For atapi devices, the
 *      first contained SGE holds the PACKET and second contained
 *      SGE holds the link to an external SGT. For non-atapi devices,
 *      the first contained SGE works as link to external SGT while
 *      second SGE is blank.
 *
 * external SGT tables: The external SGT tables pointed to from
 *      within si_prb_t are actually abstracted as si_sgblock_t. Each
 *      si_sgblock_t contains si_dma_sg_number number of
 *      SGT tables linked in a chain. Currently this default value of
 *      SGT tables per block is at 85 as  which translates
 *      to a maximum of 256 dma cookies per single dma transfer.
 *      This value can be changed through the global var: si_dma_sg_number
 *      in /etc/system, the maxium is at 21844 as which translates to 65535
 *      dma cookies per single dma transfer.
 *
 *
 * III. Driver operation
 *
 * Command Issuing: We use the "indirect method of command issuance". The
 *      PRB contains the command [and atapi PACKET] and a link to the
 *      external SGT chain. We write the physical address of the PRB into
 *      command activation register. There are 31 command slots for
 *      each port. After posting a command, we remember the posted slot &
 *      the sata packet in siport_pending_tags & siport_slot_pkts[]
 *      respectively.
 *
 * Command completion: On a successful completion, intr_command_complete()
 *      receives the control. The slot_status register holds the outstanding
 *      commands. Any reading of slot_status register automatically clears
 *      the interrupt. By comparing the slot_status register contents with
 *      per port siport_pending_tags, we determine which of the previously
 *      posted commands have finished.
 *
 * Timeout handling: Every 5 seconds, the watchdog handler scans thru the
 *      pending packets. The satapkt->satapkt_hba_driver_private field is
 *      overloaded with the count of watchdog cycles a packet has survived.
 *      If a packet has not completed within satapkt->satapkt_time, it is
 *      failed with error code of SATA_PKT_TIMEOUT. There is one watchdog
 *      handler running for each instance of controller.
 *
 * Error handling: For 3124, whenever any single command has encountered
 *      an error, the whole port execution completely stalls; there is no
 *      way of canceling or aborting the particular failed command. If
 *      the port is connected to a port multiplier, we can however RESUME
 *      other non-error devices connected to the port multiplier.
 *      The only way to recover the failed commands is to either initialize
 *      the port or reset the port/device. Both port initialize and reset
 *      operations result in discarding any of pending commands on the port.
 *      All such discarded commands are sent up to framework with PKT_RESET
 *      satapkt_reason. The assumption is that framework [and sd] would
 *      retry these commands again. The failed command itself however is
 *      sent up with PKT_DEV_ERROR.
 *
 *      Here is the implementation strategy based on SiliconImage email
 *      regarding how they handle the errors for their Windows driver:
 *
 *        a) for DEVICEERROR:
 *              If the port is connected to port multiplier, then
 *               1) Resume the port
 *               2) Wait for all the non-failed commands to complete
 *               3) Perform a Port Initialize
 *
 *              If the port is not connected to port multiplier, issue
 *              a Port Initialize.
 *
 *        b) for SDBERROR: [SDBERROR means failed command is an NCQ command]
 *              Handle exactly like DEVICEERROR handling.
 *              After the Port Initialize done, do a Read Log Extended.
 *
 *        c) for SENDFISERROR:
 *              If the port is connected to port multiplier, then
 *               1) Resume the port
 *               2) Wait for all the non-failed commands to complete
 *               3) Perform a Port Initialize
 *
 *              If the port is not connected to port multiplier, issue
 *              a Device Reset.
 *
 *        d) for DATAFISERROR:
 *              If the port was executing an NCQ command, issue a Device
 *              Reset.
 *
 *              Otherwise, follow the same error recovery as DEVICEERROR.
 *
 *        e) for any other error, simply issue a Device Reset.
 *
 *      To synchronize the interactions between various control flows (e.g.
 *      error recovery, timeout handling, si_poll_timeout, incoming flow
 *      from framework etc.), the following precautions are taken care of:
 *              a) During mopping_in_progress, no more commands are
 *              accepted from the framework.
 *
 *              b) While draining the port multiplier commands, we should
 *              handle the possibility of any of the other waited commands
 *              failing (possibly with a different error code)
 *
 * Atapi handling: For atapi devices, we use the first SGE within the PRB
 *      to fill the scsi cdb while the second SGE points to external SGT.
 *
 * Queuing: Queue management is achieved external to the driver inside sd.
 *      Based on sata_hba_tran->qdepth and IDENTIFY data, the framework
 *      enables or disables the queuing. The qdepth for si3124 is 31
 *      commands.
 *
 * Port Multiplier: Enumeration of port multiplier is handled during the
 *      controller initialization and also during the a hotplug operation.
 *      Current logic takes care of situation where a port multiplier
 *      is hotplugged into a port which had a cdisk connected previously
 *      and vice versa.
 *
 * Register poll timeouts: Currently most of poll timeouts on register
 *      reads is set to 0.5 seconds except for a value of 10 seconds
 *      while reading the device signature. [Such a big timeout values
 *      for device signature were found needed during cold reboots
 *      for devices behind port multiplier].
 *
 *
 * IV. Known Issues
 *
 * 1) Currently the atapi packet length is hard coded to 12 bytes
 *      This is wrong. The framework should determine it just like they
 *      determine ad_cdb_len in legacy atapi.c. It should even reject
 *      init_pkt() for greater CDB lengths. See atapi.c. Revisit this
 *      in 2nd phase of framework project.
 *
 * 2) Do real REQUEST SENSE command instead of faking for ATAPI case.
 *
 */


#include <sys/note.h>
#include <sys/scsi/scsi.h>
#include <sys/pci.h>
#include <sys/sata/sata_hba.h>
#include <sys/sata/adapters/si3124/si3124reg.h>
#include <sys/sata/adapters/si3124/si3124var.h>
#include <sys/sdt.h>

/*
 * FMA header files
 */
#include <sys/ddifm.h>
#include <sys/fm/protocol.h>
#include <sys/fm/util.h>
#include <sys/fm/io/ddi.h>

/*
 * Function prototypes for driver entry points
 */
static  int si_attach(dev_info_t *, ddi_attach_cmd_t);
static  int si_detach(dev_info_t *, ddi_detach_cmd_t);
static  int si_getinfo(dev_info_t *, ddi_info_cmd_t, void *, void **);
static  int si_power(dev_info_t *, int, int);
static  int si_quiesce(dev_info_t *);
/*
 * Function prototypes for SATA Framework interfaces
 */
static  int si_register_sata_hba_tran(si_ctl_state_t *);
static  int si_unregister_sata_hba_tran(si_ctl_state_t *);

static  int si_tran_probe_port(dev_info_t *, sata_device_t *);
static  int si_tran_start(dev_info_t *, sata_pkt_t *spkt);
static  int si_tran_abort(dev_info_t *, sata_pkt_t *, int);
static  int si_tran_reset_dport(dev_info_t *, sata_device_t *);
static  int si_tran_hotplug_port_activate(dev_info_t *, sata_device_t *);
static  int si_tran_hotplug_port_deactivate(dev_info_t *, sata_device_t *);

/*
 * Local function prototypes
 */

static  int si_alloc_port_state(si_ctl_state_t *, int);
static  void si_dealloc_port_state(si_ctl_state_t *, int);
static  int si_alloc_sgbpool(si_ctl_state_t *, int);
static  void si_dealloc_sgbpool(si_ctl_state_t *, int);
static  int si_alloc_prbpool(si_ctl_state_t *, int);
static  void si_dealloc_prbpool(si_ctl_state_t *, int);

static void si_find_dev_signature(si_ctl_state_t *, si_port_state_t *,
                                                int, int);
static void si_poll_cmd(si_ctl_state_t *, si_port_state_t *, int, int,
                                                sata_pkt_t *);
static  int si_claim_free_slot(si_ctl_state_t *, si_port_state_t *, int);
static  int si_deliver_satapkt(si_ctl_state_t *, si_port_state_t *, int,
                                                sata_pkt_t *);

static  int si_initialize_controller(si_ctl_state_t *);
static  void si_deinitialize_controller(si_ctl_state_t *);
static void si_init_port(si_ctl_state_t *, int);
static  int si_enumerate_port_multiplier(si_ctl_state_t *,
                                                si_port_state_t *, int);
static int si_read_portmult_reg(si_ctl_state_t *, si_port_state_t *,
                                                int, int, int, uint32_t *);
static int si_write_portmult_reg(si_ctl_state_t *, si_port_state_t *,
                                                int, int, int, uint32_t);
static void si_set_sense_data(sata_pkt_t *, int);

static uint_t si_intr(caddr_t, caddr_t);
static int si_intr_command_complete(si_ctl_state_t *,
                                        si_port_state_t *, int);
static void si_schedule_intr_command_error(si_ctl_state_t *,
                                        si_port_state_t *, int);
static void si_do_intr_command_error(void *);
static int si_intr_command_error(si_ctl_state_t *,
                                        si_port_state_t *, int);
static void si_error_recovery_DEVICEERROR(si_ctl_state_t *,
                                        si_port_state_t *, int);
static void si_error_recovery_SDBERROR(si_ctl_state_t *,
                                        si_port_state_t *, int);
static void si_error_recovery_DATAFISERROR(si_ctl_state_t *,
                                        si_port_state_t *, int);
static void si_error_recovery_SENDFISERROR(si_ctl_state_t *,
                                        si_port_state_t *, int);
static void si_error_recovery_default(si_ctl_state_t *,
                                        si_port_state_t *, int);
static uint8_t si_read_log_ext(si_ctl_state_t *,
                                        si_port_state_t *si_portp, int);
static void si_log_error_message(si_ctl_state_t *, int, uint32_t);
static int si_intr_port_ready(si_ctl_state_t *, si_port_state_t *, int);
static int si_intr_pwr_change(si_ctl_state_t *, si_port_state_t *, int);
static int si_intr_phy_ready_change(si_ctl_state_t *, si_port_state_t *, int);
static int si_intr_comwake_rcvd(si_ctl_state_t *, si_port_state_t *, int);
static int si_intr_unrecognised_fis(si_ctl_state_t *, si_port_state_t *, int);
static int si_intr_dev_xchanged(si_ctl_state_t *, si_port_state_t *, int);
static int si_intr_decode_err_threshold(si_ctl_state_t *,
                                        si_port_state_t *, int);
static int si_intr_crc_err_threshold(si_ctl_state_t *, si_port_state_t *, int);
static int si_intr_handshake_err_threshold(si_ctl_state_t *,
                                        si_port_state_t *, int);
static int si_intr_set_devbits_notify(si_ctl_state_t *, si_port_state_t *, int);

static  void si_enable_port_interrupts(si_ctl_state_t *, int);
static  void si_enable_all_interrupts(si_ctl_state_t *);
static  void si_disable_port_interrupts(si_ctl_state_t *, int);
static  void si_disable_all_interrupts(si_ctl_state_t *);
static  void fill_dev_sregisters(si_ctl_state_t *, int, sata_device_t *);
static  int si_add_legacy_intrs(si_ctl_state_t *);
static  int si_add_msi_intrs(si_ctl_state_t *);
static  void si_rem_intrs(si_ctl_state_t *);

static  int si_reset_dport_wait_till_ready(si_ctl_state_t *,
                                si_port_state_t *, int, int);
static int si_clear_port(si_ctl_state_t *, int);
static void si_schedule_port_initialize(si_ctl_state_t *,
                                si_port_state_t *, int);
static void si_do_initialize_port(void *);
static  int si_initialize_port_wait_till_ready(si_ctl_state_t *, int);

static void si_timeout_pkts(si_ctl_state_t *, si_port_state_t *, int, uint32_t);
static  void si_watchdog_handler(si_ctl_state_t *);

/*
 * FMA Prototypes
 */
static void si_fm_init(si_ctl_state_t *);
static void si_fm_fini(si_ctl_state_t *);
static int si_fm_error_cb(dev_info_t *, ddi_fm_error_t *, const void *);
static int si_check_acc_handle(ddi_acc_handle_t);
static int si_check_dma_handle(ddi_dma_handle_t);
static int si_check_ctl_handles(si_ctl_state_t *);
static int si_check_port_handles(si_port_state_t *);
static void si_fm_ereport(si_ctl_state_t *, char *, char *);

static  void si_log(si_ctl_state_t *, si_port_state_t *, char *, ...);

static void si_copy_out_regs(sata_cmd_t *, si_ctl_state_t *, uint8_t, uint8_t);

/*
 * DMA attributes for the data buffer
 */

static ddi_dma_attr_t buffer_dma_attr = {
        DMA_ATTR_V0,            /* dma_attr_version */
        0,                      /* dma_attr_addr_lo: lowest bus address */
        0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
        0xffffffffull,          /* dma_attr_count_max i.e. for one cookie */
        1,                      /* dma_attr_align: single byte aligned */
        1,                      /* dma_attr_burstsizes */
        1,                      /* dma_attr_minxfer */
        0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
        0xffffffffull,          /* dma_attr_seg */
        SI_DEFAULT_SGL_LENGTH,  /* dma_attr_sgllen */
        512,                    /* dma_attr_granular */
        0,                      /* dma_attr_flags */
};

/*
 * DMA attributes for incore RPB and SGT pool
 */
static ddi_dma_attr_t prb_sgt_dma_attr = {
        DMA_ATTR_V0,            /* dma_attr_version */
        0,                      /* dma_attr_addr_lo: lowest bus address */
        0xffffffffffffffffull,  /* dma_attr_addr_hi: highest bus address */
        0xffffffffull,          /* dma_attr_count_max i.e. for one cookie */
        8,                      /* dma_attr_align: quad word aligned */
        1,                      /* dma_attr_burstsizes */
        1,                      /* dma_attr_minxfer */
        0xffffffffull,          /* dma_attr_maxxfer i.e. includes all cookies */
        0xffffffffull,          /* dma_attr_seg */
        1,                      /* dma_attr_sgllen */
        1,                      /* dma_attr_granular */
        0,                      /* dma_attr_flags */
};

/* Device access attributes */
static ddi_device_acc_attr_t accattr = {
    DDI_DEVICE_ATTR_V1,
    DDI_STRUCTURE_LE_ACC,
    DDI_STRICTORDER_ACC,
    DDI_DEFAULT_ACC
};


static struct dev_ops sictl_dev_ops = {
        DEVO_REV,               /* devo_rev */
        0,                      /* refcnt  */
        si_getinfo,             /* info */
        nulldev,                /* identify */
        nulldev,                /* probe */
        si_attach,              /* attach */
        si_detach,              /* detach */
        nodev,                  /* no reset */
        (struct cb_ops *)0,     /* driver operations */
        NULL,                   /* bus operations */
        si_power,               /* power */
        si_quiesce,             /* devo_quiesce */
};

static sata_tran_hotplug_ops_t si_tran_hotplug_ops = {
        SATA_TRAN_HOTPLUG_OPS_REV_1,
        si_tran_hotplug_port_activate,
        si_tran_hotplug_port_deactivate
};


static int si_watchdog_timeout = 5; /* 5 seconds */
static int si_watchdog_tick;

extern struct mod_ops mod_driverops;

static  struct modldrv modldrv = {
        &mod_driverops, /* driverops */
        "si3124 driver",
        &sictl_dev_ops, /* driver ops */
};

static  struct modlinkage modlinkage = {
        MODREV_1,
        &modldrv,
        NULL
};


/* The following are needed for si_log() */
static kmutex_t si_log_mutex;
static char si_log_buf[SI_LOGBUF_LEN];
uint32_t si_debug_flags =
    SIDBG_ERRS|SIDBG_INIT|SIDBG_EVENT|SIDBG_TIMEOUT|SIDBG_RESET;

static int is_msi_supported = 0;

/*
 * The below global variables are tunable via /etc/system
 *
 * si_dma_sg_number
 */

int si_dma_sg_number = SI_DEFAULT_SGT_TABLES_PER_PRB;

/* Opaque state pointer to be initialized by ddi_soft_state_init() */
static void *si_statep  = NULL;

/*
 *  si3124 module initialization.
 *
 */
int
_init(void)
{
        int     error;

        error = ddi_soft_state_init(&si_statep, sizeof (si_ctl_state_t), 0);
        if (error != 0) {
                return (error);
        }

        mutex_init(&si_log_mutex, NULL, MUTEX_DRIVER, NULL);

        if ((error = sata_hba_init(&modlinkage)) != 0) {
                mutex_destroy(&si_log_mutex);
                ddi_soft_state_fini(&si_statep);
                return (error);
        }

        error = mod_install(&modlinkage);
        if (error != 0) {
                sata_hba_fini(&modlinkage);
                mutex_destroy(&si_log_mutex);
                ddi_soft_state_fini(&si_statep);
                return (error);
        }

        si_watchdog_tick = drv_usectohz((clock_t)si_watchdog_timeout * 1000000);

        return (error);
}

/*
 * si3124 module uninitialize.
 *
 */
int
_fini(void)
{
        int     error;

        error = mod_remove(&modlinkage);
        if (error != 0) {
                return (error);
        }

        /* Remove the resources allocated in _init(). */
        sata_hba_fini(&modlinkage);
        mutex_destroy(&si_log_mutex);
        ddi_soft_state_fini(&si_statep);

        return (error);
}

/*
 * _info entry point
 *
 */
int
_info(struct modinfo *modinfop)
{
        return (mod_info(&modlinkage, modinfop));
}


/*
 * The attach entry point for dev_ops.
 *
 * We initialize the controller, initialize the soft state, register
 * the interrupt handlers and then register ourselves with sata framework.
 */
static int
si_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
        si_ctl_state_t *si_ctlp;
        int instance;
        int status;
        int attach_state;
        int intr_types;
        sata_device_t sdevice;

        SIDBG(SIDBG_ENTRY, "si_attach enter", NULL);
        instance = ddi_get_instance(dip);
        attach_state = ATTACH_PROGRESS_NONE;

        switch (cmd) {

        case DDI_ATTACH:

                /* Allocate si_softc. */
                status = ddi_soft_state_zalloc(si_statep, instance);
                if (status != DDI_SUCCESS) {
                        goto err_out;
                }

                si_ctlp = ddi_get_soft_state(si_statep, instance);
                si_ctlp->sictl_devinfop = dip;

                attach_state |= ATTACH_PROGRESS_STATEP_ALLOC;

                /* Initialize FMA */
                si_ctlp->fm_capabilities = ddi_getprop(DDI_DEV_T_ANY, dip,
                    DDI_PROP_CANSLEEP | DDI_PROP_DONTPASS, "fm-capable",
                    DDI_FM_EREPORT_CAPABLE | DDI_FM_ACCCHK_CAPABLE |
                    DDI_FM_DMACHK_CAPABLE | DDI_FM_ERRCB_CAPABLE);

                si_fm_init(si_ctlp);

                attach_state |= ATTACH_PROGRESS_INIT_FMA;

                /* Configure pci config space handle. */
                status = pci_config_setup(dip, &si_ctlp->sictl_pci_conf_handle);
                if (status != DDI_SUCCESS) {
                        goto err_out;
                }

                si_ctlp->sictl_devid =
                    pci_config_get16(si_ctlp->sictl_pci_conf_handle,
                    PCI_CONF_DEVID);
                switch (si_ctlp->sictl_devid) {
                        case SI3124_DEV_ID:
                                si_ctlp->sictl_num_ports = SI3124_MAX_PORTS;
                                break;

                        case SI3132_DEV_ID:
                                si_ctlp->sictl_num_ports = SI3132_MAX_PORTS;
                                break;

                        case SI3531_DEV_ID:
                                si_ctlp->sictl_num_ports = SI3531_MAX_PORTS;
                                break;

                        default:
                                /*
                                 * Driver should not have attatched if device
                                 * ID is not already known and is supported.
                                 */
                                goto err_out;
                }

                attach_state |= ATTACH_PROGRESS_CONF_HANDLE;

                /* Now map the bar0; the bar0 contains the global registers. */
                status = ddi_regs_map_setup(dip,
                    PCI_BAR0,
                    (caddr_t *)&si_ctlp->sictl_global_addr,
                    0,
                    0,
                    &accattr,
                    &si_ctlp->sictl_global_acc_handle);
                if (status != DDI_SUCCESS) {
                        goto err_out;
                }

                attach_state |= ATTACH_PROGRESS_BAR0_MAP;

                /* Now map bar1; the bar1 contains the port registers. */
                status = ddi_regs_map_setup(dip,
                    PCI_BAR1,
                    (caddr_t *)&si_ctlp->sictl_port_addr,
                    0,
                    0,
                    &accattr,
                    &si_ctlp->sictl_port_acc_handle);
                if (status != DDI_SUCCESS) {
                        goto err_out;
                }

                attach_state |= ATTACH_PROGRESS_BAR1_MAP;

                /*
                 * Disable all the interrupts before adding interrupt
                 * handler(s). The interrupts shall be re-enabled selectively
                 * out of si_init_port().
                 */
                si_disable_all_interrupts(si_ctlp);

                /* Get supported interrupt types. */
                if (ddi_intr_get_supported_types(dip, &intr_types)
                    != DDI_SUCCESS) {
                        SIDBG_C(SIDBG_INIT, si_ctlp,
                            "ddi_intr_get_supported_types failed", NULL);
                        goto err_out;
                }

                SIDBG_C(SIDBG_INIT, si_ctlp,
                    "ddi_intr_get_supported_types() returned: 0x%x",
                    intr_types);

                if (is_msi_supported && (intr_types & DDI_INTR_TYPE_MSI)) {
                        SIDBG_C(SIDBG_INIT, si_ctlp,
                            "Using MSI interrupt type", NULL);

                        /*
                         * Try MSI first, but fall back to legacy if MSI
                         * attach fails.
                         */
                        if (si_add_msi_intrs(si_ctlp) == DDI_SUCCESS) {
                                si_ctlp->sictl_intr_type = DDI_INTR_TYPE_MSI;
                                attach_state |= ATTACH_PROGRESS_INTR_ADDED;
                                SIDBG_C(SIDBG_INIT, si_ctlp,
                                    "MSI interrupt setup done", NULL);
                        } else {
                                SIDBG_C(SIDBG_INIT, si_ctlp,
                                    "MSI registration failed "
                                    "will try Legacy interrupts", NULL);
                        }
                }

                if (!(attach_state & ATTACH_PROGRESS_INTR_ADDED) &&
                    (intr_types & DDI_INTR_TYPE_FIXED)) {
                        /*
                         * Either the MSI interrupt setup has failed or only
                         * fixed interrupts are available on the system.
                         */
                        SIDBG_C(SIDBG_INIT, si_ctlp,
                            "Using Legacy interrupt type", NULL);

                        if (si_add_legacy_intrs(si_ctlp) == DDI_SUCCESS) {
                                si_ctlp->sictl_intr_type = DDI_INTR_TYPE_FIXED;
                                attach_state |= ATTACH_PROGRESS_INTR_ADDED;
                                SIDBG_C(SIDBG_INIT, si_ctlp,
                                    "Legacy interrupt setup done", NULL);
                        } else {
                                SIDBG_C(SIDBG_INIT, si_ctlp,
                                    "legacy interrupt setup failed", NULL);
                                goto err_out;
                        }
                }

                if (!(attach_state & ATTACH_PROGRESS_INTR_ADDED)) {
                        SIDBG_C(SIDBG_INIT, si_ctlp,
                            "si3124: No interrupts registered", NULL);
                        goto err_out;
                }


                /* Initialize the mutex. */
                mutex_init(&si_ctlp->sictl_mutex, NULL, MUTEX_DRIVER,
                    (void *)(uintptr_t)si_ctlp->sictl_intr_pri);

                attach_state |= ATTACH_PROGRESS_MUTEX_INIT;

                /*
                 * Initialize the controller and driver core.
                 */
                si_ctlp->sictl_flags |= SI_ATTACH;
                status = si_initialize_controller(si_ctlp);
                si_ctlp->sictl_flags &= ~SI_ATTACH;
                if (status) {
                        goto err_out;
                }

                attach_state |= ATTACH_PROGRESS_HW_INIT;

                if (si_register_sata_hba_tran(si_ctlp)) {
                        SIDBG_C(SIDBG_INIT, si_ctlp,
                            "si3124: setting sata hba tran failed", NULL);
                        goto err_out;
                }

                si_ctlp->sictl_timeout_id = timeout(
                    (void (*)(void *))si_watchdog_handler,
                    (caddr_t)si_ctlp, si_watchdog_tick);

                si_ctlp->sictl_power_level = PM_LEVEL_D0;

                return (DDI_SUCCESS);

        case DDI_RESUME:
                si_ctlp = ddi_get_soft_state(si_statep, instance);

                status = si_initialize_controller(si_ctlp);
                if (status) {
                        return (DDI_FAILURE);
                }

                si_ctlp->sictl_timeout_id = timeout(
                    (void (*)(void *))si_watchdog_handler,
                    (caddr_t)si_ctlp, si_watchdog_tick);

                (void) pm_power_has_changed(dip, 0, PM_LEVEL_D0);

                /* Notify SATA framework about RESUME. */
                if (sata_hba_attach(si_ctlp->sictl_devinfop,
                    si_ctlp->sictl_sata_hba_tran,
                    DDI_RESUME) != DDI_SUCCESS) {
                        return (DDI_FAILURE);
                }

                /*
                 * Notify the "framework" that it should reprobe ports to see
                 * if any device got changed while suspended.
                 */
                bzero((void *)&sdevice, sizeof (sata_device_t));
                sata_hba_event_notify(dip, &sdevice,
                    SATA_EVNT_PWR_LEVEL_CHANGED);
                SIDBG_C(SIDBG_INIT|SIDBG_EVENT, si_ctlp,
                    "sending event up: SATA_EVNT_PWR_LEVEL_CHANGED", NULL);

                (void) pm_idle_component(si_ctlp->sictl_devinfop, 0);

                si_ctlp->sictl_power_level = PM_LEVEL_D0;

                return (DDI_SUCCESS);

        default:
                return (DDI_FAILURE);

        }

err_out:
        if (attach_state & ATTACH_PROGRESS_HW_INIT) {
                si_ctlp->sictl_flags |= SI_DETACH;
                /* We want to set SI_DETACH to deallocate all memory */
                si_deinitialize_controller(si_ctlp);
                si_ctlp->sictl_flags &= ~SI_DETACH;
        }

        if (attach_state & ATTACH_PROGRESS_MUTEX_INIT) {
                mutex_destroy(&si_ctlp->sictl_mutex);
        }

        if (attach_state & ATTACH_PROGRESS_INTR_ADDED) {
                si_rem_intrs(si_ctlp);
        }

        if (attach_state & ATTACH_PROGRESS_BAR1_MAP) {
                ddi_regs_map_free(&si_ctlp->sictl_port_acc_handle);
        }

        if (attach_state & ATTACH_PROGRESS_BAR0_MAP) {
                ddi_regs_map_free(&si_ctlp->sictl_global_acc_handle);
        }

        if (attach_state & ATTACH_PROGRESS_CONF_HANDLE) {
                pci_config_teardown(&si_ctlp->sictl_pci_conf_handle);
        }

        if (attach_state & ATTACH_PROGRESS_INIT_FMA) {
                si_fm_fini(si_ctlp);
        }

        if (attach_state & ATTACH_PROGRESS_STATEP_ALLOC) {
                ddi_soft_state_free(si_statep, instance);
        }

        return (DDI_FAILURE);
}


/*
 * The detach entry point for dev_ops.
 *
 * We undo the things we did in si_attach().
 */
static int
si_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
        si_ctl_state_t *si_ctlp;
        int instance;

        SIDBG(SIDBG_ENTRY, "si_detach enter", NULL);
        instance = ddi_get_instance(dip);
        si_ctlp = ddi_get_soft_state(si_statep, instance);

        switch (cmd) {

        case DDI_DETACH:

                mutex_enter(&si_ctlp->sictl_mutex);

                /* disable the interrupts for an uninterrupted detach */
                si_disable_all_interrupts(si_ctlp);

                mutex_exit(&si_ctlp->sictl_mutex);
                /* unregister from the sata framework. */
                if (si_unregister_sata_hba_tran(si_ctlp) != SI_SUCCESS) {
                        si_enable_all_interrupts(si_ctlp);
                        return (DDI_FAILURE);
                }
                mutex_enter(&si_ctlp->sictl_mutex);

                /* now cancel the timeout handler. */
                si_ctlp->sictl_flags |= SI_NO_TIMEOUTS;
                (void) untimeout(si_ctlp->sictl_timeout_id);
                si_ctlp->sictl_flags &= ~SI_NO_TIMEOUTS;

                /* de-initialize the controller. */
                si_ctlp->sictl_flags |= SI_DETACH;
                si_deinitialize_controller(si_ctlp);
                si_ctlp->sictl_flags &= ~SI_DETACH;

                /* destroy any mutexes */
                mutex_exit(&si_ctlp->sictl_mutex);
                mutex_destroy(&si_ctlp->sictl_mutex);

                /* remove the interrupts */
                si_rem_intrs(si_ctlp);

                /* remove the reg maps. */
                ddi_regs_map_free(&si_ctlp->sictl_port_acc_handle);
                ddi_regs_map_free(&si_ctlp->sictl_global_acc_handle);
                pci_config_teardown(&si_ctlp->sictl_pci_conf_handle);

                /* deinit FMA */
                si_fm_fini(si_ctlp);

                /* free the soft state. */
                ddi_soft_state_free(si_statep, instance);

                return (DDI_SUCCESS);

        case DDI_SUSPEND:
                /* Inform SATA framework */
                if (sata_hba_detach(dip, cmd) != DDI_SUCCESS) {
                        return (DDI_FAILURE);
                }

                mutex_enter(&si_ctlp->sictl_mutex);

                /*
                 * Device needs to be at full power in case it is needed to
                 * handle dump(9e) to save CPR state after DDI_SUSPEND
                 * completes.  This is OK since presumably power will be
                 * removed anyways.  No outstanding transactions should be
                 * on the controller since the children are already quiesced.
                 *
                 * If any ioctls/cfgadm support is added that touches
                 * hardware, those entry points will need to check for
                 * suspend and then block or return errors until resume.
                 *
                 */
                if (pm_busy_component(si_ctlp->sictl_devinfop, 0) ==
                    DDI_SUCCESS) {
                        mutex_exit(&si_ctlp->sictl_mutex);
                        (void) pm_raise_power(si_ctlp->sictl_devinfop, 0,
                            PM_LEVEL_D0);
                        mutex_enter(&si_ctlp->sictl_mutex);
                }

                si_deinitialize_controller(si_ctlp);

                si_ctlp->sictl_flags |= SI_NO_TIMEOUTS;
                (void) untimeout(si_ctlp->sictl_timeout_id);
                si_ctlp->sictl_flags &= ~SI_NO_TIMEOUTS;

                SIDBG_C(SIDBG_POWER, si_ctlp, "si3124%d: DDI_SUSPEND",
                    instance);

                mutex_exit(&si_ctlp->sictl_mutex);

                return (DDI_SUCCESS);

        default:
                return (DDI_FAILURE);

        }

}

static int
si_power(dev_info_t *dip, int component, int level)
{
#ifndef __lock_lint
        _NOTE(ARGUNUSED(component))
#endif /* __lock_lint */

        si_ctl_state_t *si_ctlp;
        int instance = ddi_get_instance(dip);
        int rval = DDI_SUCCESS;
        int old_level;
        sata_device_t sdevice;

        si_ctlp = ddi_get_soft_state(si_statep, instance);

        if (si_ctlp == NULL) {
                return (DDI_FAILURE);
        }

        SIDBG_C(SIDBG_ENTRY, si_ctlp, "si_power enter", NULL);

        mutex_enter(&si_ctlp->sictl_mutex);
        old_level = si_ctlp->sictl_power_level;

        switch (level) {
        case PM_LEVEL_D0: /* fully on */
                pci_config_put16(si_ctlp->sictl_pci_conf_handle,
                    PM_CSR(si_ctlp->sictl_devid), PCI_PMCSR_D0);
#ifndef __lock_lint
                delay(drv_usectohz(10000));
#endif  /* __lock_lint */
                si_ctlp->sictl_power_level = PM_LEVEL_D0;
                (void) pci_restore_config_regs(si_ctlp->sictl_devinfop);

                SIDBG_C(SIDBG_POWER, si_ctlp,
                    "si3124%d: turning power ON. old level %d",
                    instance, old_level);
                /*
                 * If called from attach, just raise device power,
                 * restore config registers (if they were saved
                 * from a previous detach that lowered power),
                 * and exit.
                 */
                if (si_ctlp->sictl_flags & SI_ATTACH)
                        break;

                mutex_exit(&si_ctlp->sictl_mutex);
                (void) si_initialize_controller(si_ctlp);
                mutex_enter(&si_ctlp->sictl_mutex);

                si_ctlp->sictl_timeout_id = timeout(
                    (void (*)(void *))si_watchdog_handler,
                    (caddr_t)si_ctlp, si_watchdog_tick);

                bzero((void *)&sdevice, sizeof (sata_device_t));
                sata_hba_event_notify(
                    si_ctlp->sictl_sata_hba_tran->sata_tran_hba_dip,
                    &sdevice, SATA_EVNT_PWR_LEVEL_CHANGED);
                SIDBG_C(SIDBG_EVENT|SIDBG_POWER, si_ctlp,
                    "sending event up: PWR_LEVEL_CHANGED", NULL);

                break;

        case PM_LEVEL_D3: /* fully off */
                if (!(si_ctlp->sictl_flags & SI_DETACH)) {
                        si_ctlp->sictl_flags |= SI_NO_TIMEOUTS;
                        (void) untimeout(si_ctlp->sictl_timeout_id);
                        si_ctlp->sictl_flags &= ~SI_NO_TIMEOUTS;

                        si_deinitialize_controller(si_ctlp);

                        si_ctlp->sictl_power_level = PM_LEVEL_D3;
                }

                (void) pci_save_config_regs(si_ctlp->sictl_devinfop);

                pci_config_put16(si_ctlp->sictl_pci_conf_handle,
                    PM_CSR(si_ctlp->sictl_devid), PCI_PMCSR_D3HOT);

                SIDBG_C(SIDBG_POWER, si_ctlp, "si3124%d: turning power OFF. "
                    "old level %d", instance, old_level);

                break;

        default:
                SIDBG_C(SIDBG_POWER, si_ctlp, "si3124%d: turning power OFF. "
                    "old level %d", instance, old_level);
                rval = DDI_FAILURE;
                break;
        }

        mutex_exit(&si_ctlp->sictl_mutex);

        return (rval);
}


/*
 * The info entry point for dev_ops.
 *
 */
static int
si_getinfo(dev_info_t *dip, ddi_info_cmd_t infocmd, void *arg, void **result)
{
#ifndef __lock_lint
        _NOTE(ARGUNUSED(dip))
#endif /* __lock_lint */
        si_ctl_state_t *si_ctlp;
        int instance;
        dev_t dev;

        dev = (dev_t)arg;
        instance = getminor(dev);

        switch (infocmd) {
                case DDI_INFO_DEVT2DEVINFO:
                        si_ctlp = ddi_get_soft_state(si_statep,  instance);
                        if (si_ctlp != NULL) {
                                *result = si_ctlp->sictl_devinfop;
                                return (DDI_SUCCESS);
                        } else {
                                *result = NULL;
                                return (DDI_FAILURE);
                        }
                case DDI_INFO_DEVT2INSTANCE:
                        *(int *)result = instance;
                        break;
                default:
                        break;
        }
        return (DDI_SUCCESS);
}



/*
 * Registers the si3124 with sata framework.
 */
static int
si_register_sata_hba_tran(si_ctl_state_t *si_ctlp)
{
        struct  sata_hba_tran   *sata_hba_tran;

        SIDBG_C(SIDBG_ENTRY, si_ctlp,
            "si_register_sata_hba_tran entry", NULL);

        mutex_enter(&si_ctlp->sictl_mutex);

        /* Allocate memory for the sata_hba_tran  */
        sata_hba_tran = kmem_zalloc(sizeof (sata_hba_tran_t), KM_SLEEP);

        sata_hba_tran->sata_tran_hba_rev = SATA_TRAN_HBA_REV;
        sata_hba_tran->sata_tran_hba_dip = si_ctlp->sictl_devinfop;

        if (si_dma_sg_number > SI_MAX_SGT_TABLES_PER_PRB) {
                si_dma_sg_number = SI_MAX_SGT_TABLES_PER_PRB;
        } else if (si_dma_sg_number < SI_MIN_SGT_TABLES_PER_PRB) {
                si_dma_sg_number = SI_MIN_SGT_TABLES_PER_PRB;
        }

        if (si_dma_sg_number != SI_DEFAULT_SGT_TABLES_PER_PRB) {
                buffer_dma_attr.dma_attr_sgllen = SGE_LENGTH(si_dma_sg_number);
        }
        sata_hba_tran->sata_tran_hba_dma_attr = &buffer_dma_attr;

        sata_hba_tran->sata_tran_hba_num_cports = si_ctlp->sictl_num_ports;
        sata_hba_tran->sata_tran_hba_features_support = 0;
        sata_hba_tran->sata_tran_hba_qdepth = SI_NUM_SLOTS;

        sata_hba_tran->sata_tran_probe_port = si_tran_probe_port;
        sata_hba_tran->sata_tran_start = si_tran_start;
        sata_hba_tran->sata_tran_abort = si_tran_abort;
        sata_hba_tran->sata_tran_reset_dport = si_tran_reset_dport;
        sata_hba_tran->sata_tran_selftest = NULL;
        sata_hba_tran->sata_tran_hotplug_ops = &si_tran_hotplug_ops;
        sata_hba_tran->sata_tran_pwrmgt_ops = NULL;
        sata_hba_tran->sata_tran_ioctl = NULL;
        mutex_exit(&si_ctlp->sictl_mutex);

        /* Attach it to SATA framework */
        if (sata_hba_attach(si_ctlp->sictl_devinfop, sata_hba_tran, DDI_ATTACH)
            != DDI_SUCCESS) {
                kmem_free((void *)sata_hba_tran, sizeof (sata_hba_tran_t));
                return (SI_FAILURE);
        }

        mutex_enter(&si_ctlp->sictl_mutex);
        si_ctlp->sictl_sata_hba_tran = sata_hba_tran;
        mutex_exit(&si_ctlp->sictl_mutex);

        return (SI_SUCCESS);
}


/*
 * Unregisters the si3124 with sata framework.
 */
static int
si_unregister_sata_hba_tran(si_ctl_state_t *si_ctlp)
{

        /* Detach from the SATA framework. */
        if (sata_hba_detach(si_ctlp->sictl_devinfop, DDI_DETACH) !=
            DDI_SUCCESS) {
                return (SI_FAILURE);
        }

        /* Deallocate sata_hba_tran. */
        kmem_free((void *)si_ctlp->sictl_sata_hba_tran,
            sizeof (sata_hba_tran_t));

        si_ctlp->sictl_sata_hba_tran = NULL;

        return (SI_SUCCESS);
}

/*
 * Called by sata framework to probe a port. We return the
 * cached information from a previous hardware probe.
 *
 * The actual hardware probing itself was done either from within
 * si_initialize_controller() during the driver attach or
 * from a phy ready change interrupt handler.
 */
static int
si_tran_probe_port(dev_info_t *dip, sata_device_t *sd)
{

        si_ctl_state_t  *si_ctlp;
        uint8_t cport = sd->satadev_addr.cport;
        uint8_t pmport = sd->satadev_addr.pmport;
        uint8_t qual = sd->satadev_addr.qual;
        uint8_t port_type;
        si_port_state_t *si_portp;
        si_portmult_state_t *si_portmultp;

        si_ctlp = ddi_get_soft_state(si_statep, ddi_get_instance(dip));

        SIDBG_C(SIDBG_ENTRY, si_ctlp,
            "si_tran_probe_port: cport: 0x%x, pmport: 0x%x, qual: 0x%x",
            cport, pmport, qual);

        if (cport >= SI_MAX_PORTS) {
                sd->satadev_type = SATA_DTYPE_NONE;
                sd->satadev_state = SATA_STATE_UNKNOWN; /* invalid port */
                return (SATA_FAILURE);
        }

        mutex_enter(&si_ctlp->sictl_mutex);
        si_portp = si_ctlp->sictl_ports[cport];
        mutex_exit(&si_ctlp->sictl_mutex);
        if (si_portp == NULL) {
                sd->satadev_type = SATA_DTYPE_NONE;
                sd->satadev_state = SATA_STATE_UNKNOWN;
                return (SATA_FAILURE);
        }

        mutex_enter(&si_portp->siport_mutex);

        if (qual == SATA_ADDR_PMPORT) {
                if (pmport >= si_portp->siport_portmult_state.sipm_num_ports) {
                        sd->satadev_type = SATA_DTYPE_NONE;
                        sd->satadev_state = SATA_STATE_UNKNOWN;
                        mutex_exit(&si_portp->siport_mutex);
                        return (SATA_FAILURE);
                } else {
                        si_portmultp = &si_portp->siport_portmult_state;
                        port_type = si_portmultp->sipm_port_type[pmport];
                }
        } else {
                port_type = si_portp->siport_port_type;
        }

        switch (port_type) {

        case PORT_TYPE_DISK:
                sd->satadev_type = SATA_DTYPE_ATADISK;
                break;

        case PORT_TYPE_ATAPI:
                sd->satadev_type = SATA_DTYPE_ATAPICD;
                break;

        case PORT_TYPE_MULTIPLIER:
                sd->satadev_type = SATA_DTYPE_PMULT;
                sd->satadev_add_info =
                    si_portp->siport_portmult_state.sipm_num_ports;
                break;

        case PORT_TYPE_UNKNOWN:
                sd->satadev_type = SATA_DTYPE_UNKNOWN;
                break;

        default:
                /* we don't support any other device types. */
                sd->satadev_type = SATA_DTYPE_NONE;
                break;
        }
        sd->satadev_state = SATA_STATE_READY;

        if (qual == SATA_ADDR_PMPORT) {
                (void) si_read_portmult_reg(si_ctlp, si_portp, cport,
                    pmport, PSCR_REG0, &sd->satadev_scr.sstatus);
                (void) si_read_portmult_reg(si_ctlp, si_portp, cport,
                    pmport, PSCR_REG1, &sd->satadev_scr.serror);
                (void) si_read_portmult_reg(si_ctlp, si_portp, cport,
                    pmport, PSCR_REG2, &sd->satadev_scr.scontrol);
                (void) si_read_portmult_reg(si_ctlp, si_portp, cport,
                    pmport, PSCR_REG3, &sd->satadev_scr.sactive);
        } else {
                fill_dev_sregisters(si_ctlp, cport, sd);
                if (!(si_portp->siport_active)) {
                        /*
                         * Since we are implementing the port deactivation
                         * in software only, we need to fake a valid value
                         * for sstatus when the device is in deactivated state.
                         */
                        SSTATUS_SET_DET(sd->satadev_scr.sstatus,
                            SSTATUS_DET_PHYOFFLINE);
                        SSTATUS_SET_IPM(sd->satadev_scr.sstatus,
                            SSTATUS_IPM_NODEV_NOPHY);
                        sd->satadev_state = SATA_PSTATE_SHUTDOWN;
                }
        }

        mutex_exit(&si_portp->siport_mutex);
        return (SATA_SUCCESS);
}

/*
 * Called by sata framework to transport a sata packet down stream.
 *
 * The actual work of building the FIS & transporting it to the hardware
 * is done out of the subroutine si_deliver_satapkt().
 */
static int
si_tran_start(dev_info_t *dip, sata_pkt_t *spkt)
{
        si_ctl_state_t *si_ctlp;
        uint8_t cport;
        si_port_state_t *si_portp;
        int slot;

        cport = spkt->satapkt_device.satadev_addr.cport;
        si_ctlp = ddi_get_soft_state(si_statep, ddi_get_instance(dip));
        mutex_enter(&si_ctlp->sictl_mutex);
        si_portp = si_ctlp->sictl_ports[cport];
        mutex_exit(&si_ctlp->sictl_mutex);

        SIDBG_P(SIDBG_ENTRY, si_portp,
            "si_tran_start entry", NULL);

        mutex_enter(&si_portp->siport_mutex);

        if ((si_portp->siport_port_type == PORT_TYPE_NODEV) ||
            !si_portp->siport_active) {
                /*
                 * si_intr_phy_ready_change() may have rendered it to
                 * PORT_TYPE_NODEV. cfgadm operation may have rendered
                 * it inactive.
                 */
                spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
                fill_dev_sregisters(si_ctlp, cport, &spkt->satapkt_device);
                mutex_exit(&si_portp->siport_mutex);
                return (SATA_TRAN_PORT_ERROR);
        }

        if (spkt->satapkt_cmd.satacmd_flags.sata_clear_dev_reset) {
                si_portp->siport_reset_in_progress = 0;
                SIDBG_P(SIDBG_RESET, si_portp,
                    "si_tran_start clearing the "
                    "reset_in_progress for port", NULL);
        }

        if (si_portp->siport_reset_in_progress &&
            ! spkt->satapkt_cmd.satacmd_flags.sata_ignore_dev_reset &&
            ! ddi_in_panic()) {

                spkt->satapkt_reason = SATA_PKT_BUSY;
                SIDBG_P(SIDBG_RESET, si_portp,
                    "si_tran_start returning BUSY while "
                    "reset in progress for port", NULL);
                mutex_exit(&si_portp->siport_mutex);
                return (SATA_TRAN_BUSY);
        }

        if (si_portp->mopping_in_progress > 0) {
                spkt->satapkt_reason = SATA_PKT_BUSY;
                SIDBG_P(SIDBG_RESET, si_portp,
                    "si_tran_start returning BUSY while "
                    "mopping in progress for port", NULL);
                mutex_exit(&si_portp->siport_mutex);
                return (SATA_TRAN_BUSY);
        }

        if ((slot = si_deliver_satapkt(si_ctlp, si_portp, cport, spkt))
            == SI_FAILURE) {
                spkt->satapkt_reason = SATA_PKT_QUEUE_FULL;
                SIDBG_P(SIDBG_ERRS, si_portp,
                    "si_tran_start returning QUEUE_FULL",
                    NULL);
                mutex_exit(&si_portp->siport_mutex);
                return (SATA_TRAN_QUEUE_FULL);
        }

        if (spkt->satapkt_op_mode & (SATA_OPMODE_POLLING|SATA_OPMODE_SYNCH)) {
                /* we need to poll now */
                si_poll_cmd(si_ctlp, si_portp, cport, slot, spkt);
                /*
                 * The command has completed, and spkt will be freed by the
                 * sata module, so don't keep a pointer to it lying around.
                 */
                si_portp->siport_slot_pkts[slot] = NULL;
        }

        mutex_exit(&si_portp->siport_mutex);
        return (SATA_TRAN_ACCEPTED);
}

#define SENDUP_PACKET(si_portp, satapkt, reason)                        \
        if (satapkt) {                                                  \
                if ((satapkt->satapkt_cmd.satacmd_cmd_reg ==            \
                                        SATAC_WRITE_FPDMA_QUEUED) ||    \
                    (satapkt->satapkt_cmd.satacmd_cmd_reg ==            \
                                        SATAC_READ_FPDMA_QUEUED)) {     \
                        si_portp->siport_pending_ncq_count--;           \
                }                                                       \
                satapkt->satapkt_reason = reason;                       \
                /*                                                      \
                 * We set the satapkt_reason in both synch and          \
                 * non-synch cases.                                     \
                 */                                                     \
                if (!(satapkt->satapkt_op_mode & SATA_OPMODE_SYNCH) &&  \
                        satapkt->satapkt_comp) {                        \
                        mutex_exit(&si_portp->siport_mutex);            \
                        (*satapkt->satapkt_comp)(satapkt);              \
                        mutex_enter(&si_portp->siport_mutex);           \
                }                                                       \
        }

/*
 * Mopping is necessitated because of the si3124 hardware limitation.
 * The only way to recover from errors or to abort a command is to
 * reset the port/device but such a reset also results in throwing
 * away all the unfinished pending commands.
 *
 * A port or device is reset in four scenarios:
 *      a) some commands failed with errors
 *      b) or we need to timeout some commands
 *      c) or we need to abort some commands
 *      d) or we need reset the port at the request of sata framework
 *
 * In all these scenarios, we need to send any pending unfinished
 * commands up to sata framework.
 *
 * WARNING!!! siport_mutex should be acquired before the function is called.
 */
static void
si_mop_commands(si_ctl_state_t *si_ctlp, si_port_state_t *si_portp,
    uint8_t port, uint32_t slot_status, uint32_t failed_tags,
    uint32_t timedout_tags, uint32_t aborting_tags, uint32_t reset_tags)
{
        uint32_t finished_tags, unfinished_tags;
        int tmpslot;
        sata_pkt_t *satapkt;
        struct sata_cmd_flags *flagsp;

        SIDBG_P(SIDBG_ERRS, si_portp,
            "si_mop_commands entered: slot_status: 0x%x",
            slot_status);

        SIDBG_P(SIDBG_ERRS, si_portp,
            "si_mop_commands: failed_tags: 0x%x, timedout_tags: 0x%x"
            "aborting_tags: 0x%x, reset_tags: 0x%x",
            failed_tags,
            timedout_tags,
            aborting_tags,
            reset_tags);

        /*
         * We could be here for four reasons: abort, reset,
         * timeout or error handling. Only one such mopping
         * is allowed at a time.
         */

        finished_tags =  si_portp->siport_pending_tags &
            ~slot_status & SI_SLOT_MASK;

        unfinished_tags = slot_status & SI_SLOT_MASK &
            ~failed_tags &
            ~aborting_tags &
            ~reset_tags &
            ~timedout_tags;

        /* Send up the finished_tags with SATA_PKT_COMPLETED. */
        while (finished_tags) {
                tmpslot = ddi_ffs(finished_tags) - 1;
                if (tmpslot == -1) {
                        break;
                }

                satapkt = si_portp->siport_slot_pkts[tmpslot];

                if (satapkt != NULL &&
                    satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
                        si_copy_out_regs(&satapkt->satapkt_cmd, si_ctlp,
                            port, tmpslot);
                }

                SIDBG_P(SIDBG_ERRS, si_portp,
                    "si_mop_commands sending up completed satapkt: %x",
                    satapkt);

                CLEAR_BIT(si_portp->siport_pending_tags, tmpslot);
                CLEAR_BIT(finished_tags, tmpslot);
                SENDUP_PACKET(si_portp, satapkt, SATA_PKT_COMPLETED);
        }

        ASSERT(finished_tags == 0);

        /* Send up failed_tags with SATA_PKT_DEV_ERROR. */
        while (failed_tags) {
                tmpslot = ddi_ffs(failed_tags) - 1;
                if (tmpslot == -1) {
                        break;
                }
                SIDBG_P(SIDBG_ERRS, si_portp, "si3124: si_mop_commands: "
                    "handling failed slot: 0x%x", tmpslot);

                satapkt = si_portp->siport_slot_pkts[tmpslot];

                if (satapkt != NULL) {

                        if (satapkt->satapkt_device.satadev_type ==
                            SATA_DTYPE_ATAPICD) {
                                si_set_sense_data(satapkt, SATA_PKT_DEV_ERROR);
                        }


                        flagsp = &satapkt->satapkt_cmd.satacmd_flags;

                        flagsp->sata_copy_out_lba_low_msb = B_TRUE;
                        flagsp->sata_copy_out_lba_mid_msb = B_TRUE;
                        flagsp->sata_copy_out_lba_high_msb = B_TRUE;
                        flagsp->sata_copy_out_lba_low_lsb = B_TRUE;
                        flagsp->sata_copy_out_lba_mid_lsb = B_TRUE;
                        flagsp->sata_copy_out_lba_high_lsb = B_TRUE;
                        flagsp->sata_copy_out_error_reg = B_TRUE;
                        flagsp->sata_copy_out_sec_count_msb = B_TRUE;
                        flagsp->sata_copy_out_sec_count_lsb = B_TRUE;
                        flagsp->sata_copy_out_device_reg = B_TRUE;

                        si_copy_out_regs(&satapkt->satapkt_cmd, si_ctlp,
                            port, tmpslot);

                        /*
                         * In the case of NCQ command failures, the error is
                         * overwritten by the one obtained from issuing of a
                         * READ LOG EXTENDED command.
                         */
                        if (si_portp->siport_err_tags_SDBERROR &
                            (1 << tmpslot)) {
                                satapkt->satapkt_cmd.satacmd_error_reg =
                                    si_read_log_ext(si_ctlp, si_portp, port);
                        }
                }

                CLEAR_BIT(failed_tags, tmpslot);
                CLEAR_BIT(si_portp->siport_pending_tags, tmpslot);
                SENDUP_PACKET(si_portp, satapkt, SATA_PKT_DEV_ERROR);
        }

        ASSERT(failed_tags == 0);

        /* Send up timedout_tags with SATA_PKT_TIMEOUT. */
        while (timedout_tags) {
                tmpslot = ddi_ffs(timedout_tags) - 1;
                if (tmpslot == -1) {
                        break;
                }

                satapkt = si_portp->siport_slot_pkts[tmpslot];
                SIDBG_P(SIDBG_ERRS, si_portp,
                    "si_mop_commands sending "
                    "spkt up with PKT_TIMEOUT: %x",
                    satapkt);

                CLEAR_BIT(si_portp->siport_pending_tags, tmpslot);
                CLEAR_BIT(timedout_tags, tmpslot);
                SENDUP_PACKET(si_portp, satapkt, SATA_PKT_TIMEOUT);
        }

        ASSERT(timedout_tags == 0);

        /* Send up aborting packets with SATA_PKT_ABORTED. */
        while (aborting_tags) {
                tmpslot = ddi_ffs(aborting_tags) - 1;
                if (tmpslot == -1) {
                        break;
                }

                satapkt = si_portp->siport_slot_pkts[tmpslot];
                SIDBG_P(SIDBG_ERRS, si_portp,
                    "si_mop_commands aborting spkt: %x",
                    satapkt);
                if (satapkt != NULL && satapkt->satapkt_device.satadev_type ==
                    SATA_DTYPE_ATAPICD) {
                        si_set_sense_data(satapkt, SATA_PKT_ABORTED);
                }

                CLEAR_BIT(si_portp->siport_pending_tags, tmpslot);
                CLEAR_BIT(aborting_tags, tmpslot);
                SENDUP_PACKET(si_portp, satapkt, SATA_PKT_ABORTED);

        }

        ASSERT(aborting_tags == 0);

        /* Reset tags are sent up to framework with SATA_PKT_RESET. */
        while (reset_tags) {
                tmpslot = ddi_ffs(reset_tags) - 1;
                if (tmpslot == -1) {
                        break;
                }
                satapkt = si_portp->siport_slot_pkts[tmpslot];
                SIDBG_P(SIDBG_ERRS, si_portp,
                    "si_mop_commands sending PKT_RESET for "
                    "reset spkt: %x",
                    satapkt);

                CLEAR_BIT(reset_tags, tmpslot);
                CLEAR_BIT(si_portp->siport_pending_tags, tmpslot);
                SENDUP_PACKET(si_portp, satapkt, SATA_PKT_RESET);
        }

        ASSERT(reset_tags == 0);

        /* Send up the unfinished_tags with SATA_PKT_RESET. */
        while (unfinished_tags) {
                tmpslot = ddi_ffs(unfinished_tags) - 1;
                if (tmpslot == -1) {
                        break;
                }
                satapkt = si_portp->siport_slot_pkts[tmpslot];
                SIDBG_P(SIDBG_ERRS, si_portp,
                    "si_mop_commands sending SATA_PKT_RESET for "
                    "retry spkt: %x",
                    satapkt);

                CLEAR_BIT(unfinished_tags, tmpslot);
                CLEAR_BIT(si_portp->siport_pending_tags, tmpslot);
                SENDUP_PACKET(si_portp, satapkt, SATA_PKT_RESET);
        }

        ASSERT(unfinished_tags == 0);

        si_portp->mopping_in_progress--;
        ASSERT(si_portp->mopping_in_progress >= 0);
}

/*
 * Called by the sata framework to abort the previously sent packet(s).
 *
 * We reset the device and mop the commands on the port.
 */
static int
si_tran_abort(dev_info_t *dip, sata_pkt_t *spkt, int flag)
{
        uint32_t slot_status;
        uint8_t port;
        int tmpslot;
        uint32_t aborting_tags;
        uint32_t finished_tags;
        si_port_state_t *si_portp;
        si_ctl_state_t *si_ctlp;

        port = spkt->satapkt_device.satadev_addr.cport;
        si_ctlp = ddi_get_soft_state(si_statep, ddi_get_instance(dip));
        mutex_enter(&si_ctlp->sictl_mutex);
        si_portp = si_ctlp->sictl_ports[port];
        mutex_exit(&si_ctlp->sictl_mutex);

        SIDBG_P(SIDBG_ERRS, si_portp, "si_tran_abort on port: %x", port);

        mutex_enter(&si_portp->siport_mutex);

        /*
         * If already mopping, then no need to abort anything.
         */
        if (si_portp->mopping_in_progress > 0) {
                SIDBG_P(SIDBG_ERRS, si_portp,
                    "si_tran_abort: port %d mopping "
                    "in progress, so just return", port);
                mutex_exit(&si_portp->siport_mutex);
                return (SATA_SUCCESS);
        }

        if ((si_portp->siport_port_type == PORT_TYPE_NODEV) ||
            !si_portp->siport_active) {
                /*
                 * si_intr_phy_ready_change() may have rendered it to
                 * PORT_TYPE_NODEV. cfgadm operation may have rendered
                 * it inactive.
                 */
                spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
                fill_dev_sregisters(si_ctlp, port, &spkt->satapkt_device);
                mutex_exit(&si_portp->siport_mutex);
                return (SATA_FAILURE);
        }

        if (flag == SATA_ABORT_ALL_PACKETS) {
                aborting_tags = si_portp->siport_pending_tags;
        } else {
                /*
                 * Need to abort a single packet.
                 * Search our siport_slot_pkts[] list for matching spkt.
                 */
                aborting_tags = 0xffffffff; /* 0xffffffff is impossible tag */
                for (tmpslot = 0; tmpslot < SI_NUM_SLOTS; tmpslot++) {
                        if (si_portp->siport_slot_pkts[tmpslot] == spkt) {
                                aborting_tags = (0x1 << tmpslot);
                                break;
                        }
                }

                if (aborting_tags == 0xffffffff) {
                        /* requested packet is not on pending list. */
                        fill_dev_sregisters(si_ctlp, port,
                            &spkt->satapkt_device);
                        mutex_exit(&si_portp->siport_mutex);
                        return (SATA_FAILURE);
                }
        }

        si_portp->mopping_in_progress++;

        slot_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_SLOT_STATUS(si_ctlp, port)));
        (void) si_reset_dport_wait_till_ready(si_ctlp, si_portp,
            port, SI_DEVICE_RESET);

        /*
         * Compute which have finished and which need to be retried.
         *
         * The finished tags are siport_pending_tags minus the slot_status.
         * The aborting_tags have to be reduced by finished_tags since we
         * can't possibly abort a tag which had finished already.
         */
        finished_tags =  si_portp->siport_pending_tags &
            ~slot_status & SI_SLOT_MASK;
        aborting_tags &= ~finished_tags;

        si_mop_commands(si_ctlp,
            si_portp,
            port,
            slot_status,
            0, /* failed_tags */
            0, /* timedout_tags */
            aborting_tags,
            0); /* reset_tags */

        fill_dev_sregisters(si_ctlp, port, &spkt->satapkt_device);
        mutex_exit(&si_portp->siport_mutex);
        return (SATA_SUCCESS);
}


/*
 * Used to reject all the pending packets on a port during a reset
 * operation.
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static void
si_reject_all_reset_pkts(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        uint32_t slot_status;
        uint32_t reset_tags;

        _NOTE(ASSUMING_PROTECTED(si_portp))

        SIDBG_P(SIDBG_RESET, si_portp,
            "si_reject_all_reset_pkts on port: %x",
            port);

        slot_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_SLOT_STATUS(si_ctlp, port)));

        /* Compute which tags need to be sent up. */
        reset_tags = slot_status & SI_SLOT_MASK;

        si_portp->mopping_in_progress++;

        si_mop_commands(si_ctlp,
            si_portp,
            port,
            slot_status,
            0, /* failed_tags */
            0, /* timedout_tags */
            0, /* aborting_tags */
            reset_tags);
}


/*
 * Called by sata framework to reset a port(s) or device.
 *
 */
static int
si_tran_reset_dport(dev_info_t *dip, sata_device_t *sd)
{
        si_ctl_state_t  *si_ctlp;
        uint8_t port = sd->satadev_addr.cport;
        int i;
        si_port_state_t *si_portp;
        int retval = SI_SUCCESS;

        si_ctlp = ddi_get_soft_state(si_statep, ddi_get_instance(dip));
        SIDBG_C(SIDBG_RESET, si_ctlp,
            "si_tran_reset_port entry: port: 0x%x",
            port);

        switch (sd->satadev_addr.qual) {
        case SATA_ADDR_CPORT:
                mutex_enter(&si_ctlp->sictl_mutex);
                si_portp = si_ctlp->sictl_ports[port];
                mutex_exit(&si_ctlp->sictl_mutex);

                mutex_enter(&si_portp->siport_mutex);

                /*
                 * If already mopping, then no need to reset or mop again.
                 */
                if (si_portp->mopping_in_progress > 0) {
                        SIDBG_P(SIDBG_RESET, si_portp,
                            "si_tran_reset_dport: CPORT port %d mopping "
                            "in progress, so just return", port);
                        mutex_exit(&si_portp->siport_mutex);
                        retval = SI_SUCCESS;
                        break;
                }

                retval = si_reset_dport_wait_till_ready(si_ctlp, si_portp, port,
                    SI_PORT_RESET);
                si_reject_all_reset_pkts(si_ctlp,  si_portp, port);
                mutex_exit(&si_portp->siport_mutex);

                break;

        case SATA_ADDR_DCPORT:
                mutex_enter(&si_ctlp->sictl_mutex);
                si_portp = si_ctlp->sictl_ports[port];
                mutex_exit(&si_ctlp->sictl_mutex);

                mutex_enter(&si_portp->siport_mutex);

                if ((si_portp->siport_port_type == PORT_TYPE_NODEV) ||
                    !si_portp->siport_active) {
                        mutex_exit(&si_portp->siport_mutex);
                        retval = SI_FAILURE;
                        break;
                }

                /*
                 * If already mopping, then no need to reset or mop again.
                 */
                if (si_portp->mopping_in_progress > 0) {
                        SIDBG_P(SIDBG_RESET, si_portp,
                            "si_tran_reset_dport: DCPORT port %d mopping "
                            "in progress, so just return", port);
                        mutex_exit(&si_portp->siport_mutex);
                        retval = SI_SUCCESS;
                        break;
                }

                retval = si_reset_dport_wait_till_ready(si_ctlp, si_portp, port,
                    SI_DEVICE_RESET);
                si_reject_all_reset_pkts(si_ctlp,  si_portp, port);
                mutex_exit(&si_portp->siport_mutex);

                break;

        case SATA_ADDR_CNTRL:
                for (i = 0; i < si_ctlp->sictl_num_ports; i++) {
                        mutex_enter(&si_ctlp->sictl_mutex);
                        si_portp = si_ctlp->sictl_ports[i];
                        mutex_exit(&si_ctlp->sictl_mutex);

                        mutex_enter(&si_portp->siport_mutex);

                        /*
                         * If mopping, then all the pending commands are being
                         * mopped, therefore there is nothing else to do.
                         */
                        if (si_portp->mopping_in_progress > 0) {
                                SIDBG_P(SIDBG_RESET, si_portp,
                                    "si_tran_reset_dport: CNTRL port %d mopping"
                                    " in progress, so just return", i);
                                mutex_exit(&si_portp->siport_mutex);
                                retval = SI_SUCCESS;
                                break;
                        }

                        retval = si_reset_dport_wait_till_ready(si_ctlp,
                            si_portp, i, SI_PORT_RESET);
                        if (retval) {
                                mutex_exit(&si_portp->siport_mutex);
                                break;
                        }
                        si_reject_all_reset_pkts(si_ctlp,  si_portp, i);
                        mutex_exit(&si_portp->siport_mutex);
                }
                break;

        case SATA_ADDR_PMPORT:
        case SATA_ADDR_DPMPORT:
                SIDBG_P(SIDBG_RESET, si_portp,
                    "port mult reset not implemented yet", NULL);
                /* FALLTHROUGH */

        default:
                retval = SI_FAILURE;

        }

        return (retval);
}


/*
 * Called by sata framework to activate a port as part of hotplug.
 *
 * Note: Not port-mult aware.
 */
static int
si_tran_hotplug_port_activate(dev_info_t *dip, sata_device_t *satadev)
{
        si_ctl_state_t *si_ctlp;
        si_port_state_t *si_portp;
        uint8_t port;

        si_ctlp = ddi_get_soft_state(si_statep, ddi_get_instance(dip));
        port = satadev->satadev_addr.cport;
        mutex_enter(&si_ctlp->sictl_mutex);
        si_portp = si_ctlp->sictl_ports[port];
        mutex_exit(&si_ctlp->sictl_mutex);

        SIDBG_P(SIDBG_EVENT, si_portp, "si_tran_hotplug_port_activate entry",
            NULL);

        mutex_enter(&si_portp->siport_mutex);
        si_enable_port_interrupts(si_ctlp, port);

        /*
         * Reset the device so that a si_find_dev_signature() would trigger.
         * But this reset is an internal operation; the sata framework does
         * not need to know about it.
         */
        (void) si_reset_dport_wait_till_ready(si_ctlp, si_portp, port,
            SI_DEVICE_RESET|SI_RESET_NO_EVENTS_UP);

        satadev->satadev_state = SATA_STATE_READY;

        si_portp->siport_active = PORT_ACTIVE;

        fill_dev_sregisters(si_ctlp, port, satadev);

        mutex_exit(&si_portp->siport_mutex);
        return (SATA_SUCCESS);
}

/*
 * Called by sata framework to deactivate a port as part of hotplug.
 *
 * Note: Not port-mult aware.
 */
static int
si_tran_hotplug_port_deactivate(dev_info_t *dip, sata_device_t *satadev)
{
        si_ctl_state_t *si_ctlp;
        si_port_state_t *si_portp;
        uint8_t port;

        si_ctlp = ddi_get_soft_state(si_statep, ddi_get_instance(dip));
        port = satadev->satadev_addr.cport;
        mutex_enter(&si_ctlp->sictl_mutex);
        si_portp = si_ctlp->sictl_ports[port];
        mutex_exit(&si_ctlp->sictl_mutex);

        SIDBG(SIDBG_EVENT, "si_tran_hotplug_port_deactivate entry", NULL);

        mutex_enter(&si_portp->siport_mutex);
        if (si_portp->siport_pending_tags & SI_SLOT_MASK) {
                /*
                 * There are pending commands on this port.
                 * Fail the deactivate request.
                 */
                satadev->satadev_state = SATA_STATE_READY;
                mutex_exit(&si_portp->siport_mutex);
                return (SATA_FAILURE);
        }

        /* mark the device as not accessible any more. */
        si_portp->siport_active = PORT_INACTIVE;

        /* disable the interrupts on the port. */
        si_disable_port_interrupts(si_ctlp, port);

        satadev->satadev_state = SATA_PSTATE_SHUTDOWN;

        fill_dev_sregisters(si_ctlp, port, satadev);
        /*
         * Since we are implementing the port deactivation in software only,
         * we need to fake a valid value for sstatus.
         */
        SSTATUS_SET_DET(satadev->satadev_scr.sstatus, SSTATUS_DET_PHYOFFLINE);
        SSTATUS_SET_IPM(satadev->satadev_scr.sstatus, SSTATUS_IPM_NODEV_NOPHY);

        mutex_exit(&si_portp->siport_mutex);
        return (SATA_SUCCESS);
}


/*
 * Allocates the si_port_state_t.
 */
static int
si_alloc_port_state(si_ctl_state_t *si_ctlp, int port)
{
        si_port_state_t *si_portp;

        si_ctlp->sictl_ports[port] = (si_port_state_t *)kmem_zalloc(
            sizeof (si_port_state_t), KM_SLEEP);

        si_portp = si_ctlp->sictl_ports[port];
        mutex_init(&si_portp->siport_mutex, NULL, MUTEX_DRIVER,
            (void *)(uintptr_t)si_ctlp->sictl_intr_pri);
        mutex_enter(&si_portp->siport_mutex);

        /* allocate prb & sgt pkts for this port. */
        if (si_alloc_prbpool(si_ctlp, port)) {
                mutex_exit(&si_portp->siport_mutex);
                kmem_free(si_ctlp->sictl_ports[port], sizeof (si_port_state_t));
                return (SI_FAILURE);
        }
        if (si_alloc_sgbpool(si_ctlp, port)) {
                si_dealloc_prbpool(si_ctlp, port);
                mutex_exit(&si_portp->siport_mutex);
                kmem_free(si_ctlp->sictl_ports[port], sizeof (si_port_state_t));
                return (SI_FAILURE);
        }

        /* Allocate the argument for the timeout */
        si_portp->siport_event_args =
            kmem_zalloc(sizeof (si_event_arg_t), KM_SLEEP);

        si_portp->siport_active = PORT_ACTIVE;
        mutex_exit(&si_portp->siport_mutex);

        return (SI_SUCCESS);

}

/*
 * Deallocates the si_port_state_t.
 */
static void
si_dealloc_port_state(si_ctl_state_t *si_ctlp, int port)
{
        si_port_state_t *si_portp;
        si_portp = si_ctlp->sictl_ports[port];

        mutex_enter(&si_portp->siport_mutex);
        kmem_free(si_portp->siport_event_args, sizeof (si_event_arg_t));
        si_dealloc_sgbpool(si_ctlp, port);
        si_dealloc_prbpool(si_ctlp, port);
        mutex_exit(&si_portp->siport_mutex);

        mutex_destroy(&si_portp->siport_mutex);

        kmem_free(si_ctlp->sictl_ports[port], sizeof (si_port_state_t));

}

/*
 * Allocates the SGB (Scatter Gather Block) incore buffer.
 */
static int
si_alloc_sgbpool(si_ctl_state_t *si_ctlp, int port)
{
        si_port_state_t *si_portp;
        uint_t cookie_count;
        size_t incore_sgbpool_size = SI_NUM_SLOTS * sizeof (si_sgblock_t)
            * si_dma_sg_number;
        size_t ret_len;
        ddi_dma_cookie_t sgbpool_dma_cookie;

        si_portp = si_ctlp->sictl_ports[port];

        /* allocate sgbpool dma handle. */
        if (ddi_dma_alloc_handle(si_ctlp->sictl_devinfop,
            &prb_sgt_dma_attr,
            DDI_DMA_SLEEP,
            NULL,
            &si_portp->siport_sgbpool_dma_handle) !=
            DDI_SUCCESS) {

                return (SI_FAILURE);
        }

        /* allocate the memory for sgbpool. */
        if (ddi_dma_mem_alloc(si_portp->siport_sgbpool_dma_handle,
            incore_sgbpool_size,
            &accattr,
            DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
            DDI_DMA_SLEEP,
            NULL,
            (caddr_t *)&si_portp->siport_sgbpool,
            &ret_len,
            &si_portp->siport_sgbpool_acc_handle) != DDI_SUCCESS) {

                /*  error.. free the dma handle. */
                ddi_dma_free_handle(&si_portp->siport_sgbpool_dma_handle);
                return (SI_FAILURE);
        }

        /* now bind it */
        if (ddi_dma_addr_bind_handle(si_portp->siport_sgbpool_dma_handle,
            NULL,
            (caddr_t)si_portp->siport_sgbpool,
            incore_sgbpool_size,
            DDI_DMA_CONSISTENT,
            DDI_DMA_SLEEP,
            NULL,
            &sgbpool_dma_cookie,
            &cookie_count) !=  DDI_DMA_MAPPED) {
                /*  error.. free the dma handle & free the memory. */
                ddi_dma_mem_free(&si_portp->siport_sgbpool_acc_handle);
                ddi_dma_free_handle(&si_portp->siport_sgbpool_dma_handle);
                return (SI_FAILURE);
        }

        si_portp->siport_sgbpool_physaddr = sgbpool_dma_cookie.dmac_laddress;
        return (SI_SUCCESS);
}

/*
 * Deallocates the SGB (Scatter Gather Block) incore buffer.
 */
static void
si_dealloc_sgbpool(si_ctl_state_t *si_ctlp, int port)
{
        si_port_state_t *si_portp = si_ctlp->sictl_ports[port];

        /* Unbind the dma handle first. */
        (void) ddi_dma_unbind_handle(si_portp->siport_sgbpool_dma_handle);

        /* Then free the underlying memory. */
        ddi_dma_mem_free(&si_portp->siport_sgbpool_acc_handle);

        /* Now free the handle itself. */
        ddi_dma_free_handle(&si_portp->siport_sgbpool_dma_handle);

}

/*
 * Allocates the PRB (Port Request Block) incore packets.
 */
static int
si_alloc_prbpool(si_ctl_state_t *si_ctlp, int port)
{
        si_port_state_t *si_portp;
        uint_t cookie_count;
        size_t incore_pkt_size = SI_NUM_SLOTS * sizeof (si_prb_t);
        size_t ret_len;
        ddi_dma_cookie_t prbpool_dma_cookie;

        si_portp = si_ctlp->sictl_ports[port];

        /* allocate prb pkts. */
        if (ddi_dma_alloc_handle(si_ctlp->sictl_devinfop,
            &prb_sgt_dma_attr,
            DDI_DMA_SLEEP,
            NULL,
            &si_portp->siport_prbpool_dma_handle) !=
            DDI_SUCCESS) {

                return (SI_FAILURE);
        }

        if (ddi_dma_mem_alloc(si_portp->siport_prbpool_dma_handle,
            incore_pkt_size,
            &accattr,
            DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
            DDI_DMA_SLEEP,
            NULL,
            (caddr_t *)&si_portp->siport_prbpool,
            &ret_len,
            &si_portp->siport_prbpool_acc_handle) != DDI_SUCCESS) {

                /* error.. free the dma handle. */
                ddi_dma_free_handle(&si_portp->siport_prbpool_dma_handle);
                return (SI_FAILURE);
        }

        if (ddi_dma_addr_bind_handle(si_portp->siport_prbpool_dma_handle,
            NULL,
            (caddr_t)si_portp->siport_prbpool,
            incore_pkt_size,
            DDI_DMA_CONSISTENT,
            DDI_DMA_SLEEP,
            NULL,
            &prbpool_dma_cookie,
            &cookie_count) !=  DDI_DMA_MAPPED) {
                /*  error.. free the dma handle & free the memory. */
                ddi_dma_mem_free(&si_portp->siport_prbpool_acc_handle);
                ddi_dma_free_handle(&si_portp->siport_prbpool_dma_handle);
                return (SI_FAILURE);
        }

        si_portp->siport_prbpool_physaddr =
            prbpool_dma_cookie.dmac_laddress;
        return (SI_SUCCESS);
}

/*
 * Deallocates the PRB (Port Request Block) incore packets.
 */
static void
si_dealloc_prbpool(si_ctl_state_t *si_ctlp, int port)
{
        si_port_state_t *si_portp = si_ctlp->sictl_ports[port];

        /* Unbind the prb dma handle first. */
        (void) ddi_dma_unbind_handle(si_portp->siport_prbpool_dma_handle);

        /* Then free the underlying memory. */
        ddi_dma_mem_free(&si_portp->siport_prbpool_acc_handle);

        /* Now free the handle itself. */
        ddi_dma_free_handle(&si_portp->siport_prbpool_dma_handle);

}



/*
 * Soft-reset the port to find the signature of the device connected to
 * the port.
 */
static void
si_find_dev_signature(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port,
        int pmp)
{
        si_prb_t *prb;
        uint32_t slot_status, signature;
        int slot, loop_count;

        SIDBG_P(SIDBG_INIT, si_portp,
            "si_find_dev_signature enter: port: %x, pmp: %x",
            port, pmp);

        /* Build a Soft Reset PRB in host memory. */
        mutex_enter(&si_portp->siport_mutex);

        slot = si_claim_free_slot(si_ctlp, si_portp, port);
        if (slot == SI_FAILURE) {
                /* Empty slot could not be found. */
                if (pmp != PORTMULT_CONTROL_PORT) {
                        /* We are behind port multiplier. */
                        si_portp->siport_portmult_state.sipm_port_type[pmp] =
                            PORT_TYPE_NODEV;
                } else {
                        si_portp->siport_port_type = PORT_TYPE_NODEV;
                }

                mutex_exit(&si_portp->siport_mutex);
                return;
        }
        prb = &si_portp->siport_prbpool[slot];
        bzero((void *)prb, sizeof (si_prb_t));

        SET_FIS_PMP(prb->prb_fis, pmp);
        SET_PRB_CONTROL_SOFT_RESET(prb);

#if SI_DEBUG
        if (si_debug_flags & SIDBG_DUMP_PRB) {
                char *ptr;
                int j;

                ptr = (char *)prb;
                cmn_err(CE_WARN, "si_find_dev_signature, prb: ");
                for (j = 0; j < (sizeof (si_prb_t)); j++) {
                        if (j%4 == 0) {
                                cmn_err(CE_WARN, "----");
                        }
                        cmn_err(CE_WARN, "%x ", ptr[j]);
                }

        }
#endif /* SI_DEBUG */

        /* deliver soft reset prb to empty slot. */
        POST_PRB_ADDR(si_ctlp, si_portp, port, slot);

        loop_count = 0;
        /* Loop till the soft reset is finished. */
        do {
                slot_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)(PORT_SLOT_STATUS(si_ctlp, port)));

                if (loop_count++ > SI_POLLRATE_SOFT_RESET) {
                        /* We are effectively timing out after 10 sec. */
                        break;
                }

                /* Wait for 10 millisec */
#ifndef __lock_lint
                delay(SI_10MS_TICKS);
#endif /* __lock_lint */

        } while (slot_status & SI_SLOT_MASK & (0x1 << slot));

        SIDBG_P(SIDBG_POLL_LOOP, si_portp,
            "si_find_dev_signature: loop count: %d, slot_status: 0x%x",
            loop_count, slot_status);

        CLEAR_BIT(si_portp->siport_pending_tags, slot);

        /* Read device signature from command slot. */
        signature = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_SIGNATURE_MSB(si_ctlp, port, slot)));
        signature <<= 8;
        signature |= (0xff & ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_SIGNATURE_LSB(si_ctlp,
            port, slot))));

        SIDBG_P(SIDBG_INIT, si_portp, "Device signature: 0x%x", signature);

        if (signature == SI_SIGNATURE_PORT_MULTIPLIER) {

                SIDBG_P(SIDBG_INIT, si_portp,
                    "Found multiplier at cport: 0x%d, pmport: 0x%x",
                    port, pmp);

                if (pmp != PORTMULT_CONTROL_PORT) {
                        /*
                         * It is wrong to chain a port multiplier behind
                         * another port multiplier.
                         */
                        si_portp->siport_portmult_state.sipm_port_type[pmp] =
                            PORT_TYPE_NODEV;
                } else {
                        si_portp->siport_port_type = PORT_TYPE_MULTIPLIER;
                        mutex_exit(&si_portp->siport_mutex);
                        (void) si_enumerate_port_multiplier(si_ctlp,
                            si_portp, port);
                        mutex_enter(&si_portp->siport_mutex);
                }
                si_init_port(si_ctlp, port);

        } else if (signature == SI_SIGNATURE_ATAPI) {
                if (pmp != PORTMULT_CONTROL_PORT) {
                        /* We are behind port multiplier. */
                        si_portp->siport_portmult_state.sipm_port_type[pmp] =
                            PORT_TYPE_ATAPI;
                } else {
                        si_portp->siport_port_type = PORT_TYPE_ATAPI;
                        si_init_port(si_ctlp, port);
                }
                SIDBG_P(SIDBG_INIT, si_portp,
                    "Found atapi at : cport: %x, pmport: %x",
                    port, pmp);

        } else if (signature == SI_SIGNATURE_DISK) {

                if (pmp != PORTMULT_CONTROL_PORT) {
                        /* We are behind port multiplier. */
                        si_portp->siport_portmult_state.sipm_port_type[pmp] =
                            PORT_TYPE_DISK;
                } else {
                        si_portp->siport_port_type = PORT_TYPE_DISK;
                        si_init_port(si_ctlp, port);
                }
                SIDBG_P(SIDBG_INIT, si_portp,
                    "found disk at : cport: %x, pmport: %x",
                    port, pmp);

        } else {
                if (pmp != PORTMULT_CONTROL_PORT) {
                        /* We are behind port multiplier. */
                        si_portp->siport_portmult_state.sipm_port_type[pmp] =
                            PORT_TYPE_UNKNOWN;
                } else {
                        si_portp->siport_port_type = PORT_TYPE_UNKNOWN;
                }
                SIDBG_P(SIDBG_INIT, si_portp,
                    "Found unknown signature 0x%x at: port: %x, pmp: %x",
                    signature, port, pmp);
        }

        mutex_exit(&si_portp->siport_mutex);
}


/*
 * Polls for the completion of the command. This is safe with both
 * interrupts enabled or disabled.
 */
static void
si_poll_cmd(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port,
        int slot,
        sata_pkt_t *satapkt)
{
        uint32_t slot_status;
        int pkt_timeout_ticks;
        uint32_t port_intr_status;
        int in_panic = ddi_in_panic();

        SIDBG_P(SIDBG_ENTRY, si_portp, "si_poll_cmd entered: port: 0x%x", port);

        pkt_timeout_ticks = drv_usectohz((clock_t)satapkt->satapkt_time *
            1000000);


        /* we start out with SATA_PKT_COMPLETED as the satapkt_reason */
        satapkt->satapkt_reason = SATA_PKT_COMPLETED;

        do {
                slot_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)(PORT_SLOT_STATUS(si_ctlp, port)));

                if (slot_status & SI_SLOT_MASK & (0x1 << slot)) {
                        if (in_panic) {
                                /*
                                 * If we are in panic, we can't rely on
                                 * timers; so, busy wait instead of delay().
                                 */
                                mutex_exit(&si_portp->siport_mutex);
                                drv_usecwait(SI_1MS_USECS);
                                mutex_enter(&si_portp->siport_mutex);
                        } else {
                                mutex_exit(&si_portp->siport_mutex);
#ifndef __lock_lint
                                delay(SI_1MS_TICKS);
#endif /* __lock_lint */
                                mutex_enter(&si_portp->siport_mutex);
                        }
                } else {
                        break;
                }

                pkt_timeout_ticks -= SI_1MS_TICKS;

        } while (pkt_timeout_ticks > 0);

        if (satapkt->satapkt_reason != SATA_PKT_COMPLETED) {
                /* The si_mop_command() got to our packet before us */

                return;
        }

        /*
         * Interrupts and timers may not be working properly in a crash dump
         * situation; we may need to handle all the three conditions here:
         * successful completion, packet failure and packet timeout.
         */
        if (IS_ATTENTION_RAISED(slot_status)) { /* error seen on port */

                port_intr_status = ddi_get32(si_ctlp->sictl_global_acc_handle,
                    (uint32_t *)PORT_INTERRUPT_STATUS(si_ctlp, port));

                SIDBG_P(SIDBG_VERBOSE, si_portp,
                    "si_poll_cmd: port_intr_status: 0x%x, port: %x",
                    port_intr_status, port);

                if (port_intr_status & INTR_COMMAND_ERROR) {
                        mutex_exit(&si_portp->siport_mutex);
                        (void) si_intr_command_error(si_ctlp, si_portp, port);
                        mutex_enter(&si_portp->siport_mutex);

                        return;

                        /*
                         * Why do we need to call si_intr_command_error() ?
                         *
                         * Answer: Even if the current packet is not the
                         * offending command, we need to restart the stalled
                         * port; (may be, the interrupts are not working well
                         * in panic condition). The call to routine
                         * si_intr_command_error() will achieve that.
                         *
                         * What if the interrupts are working fine and the
                         * si_intr_command_error() gets called once more from
                         * interrupt context ?
                         *
                         * Answer: The second instance of routine
                         * si_intr_command_error() will not mop anything
                         * since the first error handler has already blown
                         * away the hardware pending queues through reset.
                         *
                         * Will the si_intr_command_error() hurt current
                         * packet ?
                         *
                         * Answer: No.
                         */
                } else {
                        /* Ignore any non-error interrupts at this stage */
                        ddi_put32(si_ctlp->sictl_port_acc_handle,
                            (uint32_t *)(PORT_INTERRUPT_STATUS(si_ctlp,
                            port)),
                            port_intr_status & INTR_MASK);
                }

        } else if (slot_status & SI_SLOT_MASK & (0x1 << slot)) {
                satapkt->satapkt_reason = SATA_PKT_TIMEOUT;

        } /* else: the command completed successfully */

        if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
                si_copy_out_regs(&satapkt->satapkt_cmd, si_ctlp, port, slot);
        }

        if ((satapkt->satapkt_cmd.satacmd_cmd_reg ==
            SATAC_WRITE_FPDMA_QUEUED) ||
            (satapkt->satapkt_cmd.satacmd_cmd_reg ==
            SATAC_READ_FPDMA_QUEUED)) {
                si_portp->siport_pending_ncq_count--;
        }

        CLEAR_BIT(si_portp->siport_pending_tags, slot);

        /*
         * tidbit: What is the interaction of abort with polling ?
         * What happens if the current polled pkt is aborted in parallel ?
         *
         * Answer: Assuming that the si_mop_commands() completes ahead
         * of polling, all it does is to set the satapkt_reason to
         * SPKT_PKT_ABORTED. That would be fine with us.
         *
         * The same logic applies to reset interacting with polling.
         */
}


/*
 * Searches for and claims a free slot.
 *
 * Returns:     SI_FAILURE if no slots found
 *              claimed slot number if successful
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
/*ARGSUSED*/
static int
si_claim_free_slot(si_ctl_state_t *si_ctlp, si_port_state_t *si_portp, int port)
{
        uint32_t free_slots;
        int slot;

        _NOTE(ASSUMING_PROTECTED(si_portp))

        SIDBG_P(SIDBG_ENTRY, si_portp,
            "si_claim_free_slot entry: siport_pending_tags: %x",
            si_portp->siport_pending_tags);

        free_slots = (~si_portp->siport_pending_tags) & SI_SLOT_MASK;
        slot = ddi_ffs(free_slots) - 1;
        if (slot == -1) {
                SIDBG_P(SIDBG_VERBOSE, si_portp,
                    "si_claim_free_slot: no empty slots", NULL);
                return (SI_FAILURE);
        }

        si_portp->siport_pending_tags |= (0x1 << slot);
        SIDBG_P(SIDBG_VERBOSE, si_portp, "si_claim_free_slot: found slot: 0x%x",
            slot);
        return (slot);
}

/*
 * Builds the PRB for the sata packet and delivers it to controller.
 *
 * Returns:
 *      slot number if we can obtain a slot successfully
 *      otherwise, return SI_FAILURE
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static int
si_deliver_satapkt(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port,
        sata_pkt_t *spkt)
{
        int slot;
        si_prb_t *prb;
        sata_cmd_t *cmd;
        si_sge_t *sgep; /* scatter gather entry pointer */
        si_sgt_t *sgtp; /* scatter gather table pointer */
        si_sgblock_t *sgbp; /* scatter gather block pointer */
        int i, j, cookie_index;
        int ncookies;
        int is_atapi = 0;
        ddi_dma_cookie_t cookie;

        _NOTE(ASSUMING_PROTECTED(si_portp))

        slot = si_claim_free_slot(si_ctlp, si_portp, port);
        if (slot == SI_FAILURE) {
                return (SI_FAILURE);
        }

        if (spkt->satapkt_device.satadev_type == SATA_DTYPE_ATAPICD) {
                is_atapi = 1;
        }

        if ((si_portp->siport_port_type == PORT_TYPE_NODEV) ||
            !si_portp->siport_active) {
                /*
                 * si_intr_phy_ready_change() may have rendered it to
                 * PORT_TYPE_NODEV. cfgadm operation may have rendered
                 * it inactive.
                 */
                spkt->satapkt_reason = SATA_PKT_PORT_ERROR;
                fill_dev_sregisters(si_ctlp, port, &spkt->satapkt_device);
                CLEAR_BIT(si_portp->siport_pending_tags, slot);

                return (SI_FAILURE);
        }


        prb =  &(si_portp->siport_prbpool[slot]);
        bzero((void *)prb, sizeof (si_prb_t));

        cmd = &spkt->satapkt_cmd;

        SIDBG_P(SIDBG_ENTRY, si_portp,
            "si_deliver_satpkt entry: cmd_reg: 0x%x, slot: 0x%x, \
                port: %x, satapkt: %x",
            cmd->satacmd_cmd_reg, slot, port, (uint32_t)(intptr_t)spkt);

        /* Now fill the prb. */
        if (is_atapi) {
                if (spkt->satapkt_cmd.satacmd_flags.sata_data_direction ==
                    SATA_DIR_READ) {
                        SET_PRB_CONTROL_PKT_READ(prb);
                } else if (spkt->satapkt_cmd.satacmd_flags.sata_data_direction
                    == SATA_DIR_WRITE) {
                        SET_PRB_CONTROL_PKT_WRITE(prb);
                }
        }

        SET_FIS_TYPE(prb->prb_fis, REGISTER_FIS_H2D);
        if ((spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_PMPORT) ||
            (spkt->satapkt_device.satadev_addr.qual == SATA_ADDR_DPMPORT)) {
                SET_FIS_PMP(prb->prb_fis,
                    spkt->satapkt_device.satadev_addr.pmport);
        }
        SET_FIS_CDMDEVCTL(prb->prb_fis, 1);
        SET_FIS_COMMAND(prb->prb_fis, cmd->satacmd_cmd_reg);
        SET_FIS_FEATURES(prb->prb_fis, cmd->satacmd_features_reg);
        SET_FIS_SECTOR_COUNT(prb->prb_fis, cmd->satacmd_sec_count_lsb);

        switch (cmd->satacmd_addr_type) {

        case 0:
                /*
                 * satacmd_addr_type will be 0 for the commands below:
                 *      SATAC_PACKET
                 *      SATAC_IDLE_IM
                 *      SATAC_STANDBY_IM
                 *      SATAC_DOWNLOAD_MICROCODE
                 *      SATAC_FLUSH_CACHE
                 *      SATAC_SET_FEATURES
                 *      SATAC_SMART
                 *      SATAC_ID_PACKET_DEVICE
                 *      SATAC_ID_DEVICE
                 *      SATAC_READ_PORTMULT
                 *      SATAC_WRITE_PORTMULT
                 */
                /* FALLTHRU */

        case ATA_ADDR_LBA:
                /* FALLTHRU */

        case ATA_ADDR_LBA28:
                /* LBA[7:0] */
                SET_FIS_SECTOR(prb->prb_fis, cmd->satacmd_lba_low_lsb);

                /* LBA[15:8] */
                SET_FIS_CYL_LOW(prb->prb_fis, cmd->satacmd_lba_mid_lsb);

                /* LBA[23:16] */
                SET_FIS_CYL_HI(prb->prb_fis, cmd->satacmd_lba_high_lsb);

                /* LBA [27:24] (also called dev_head) */
                SET_FIS_DEV_HEAD(prb->prb_fis, cmd->satacmd_device_reg);

                break;

        case ATA_ADDR_LBA48:
                /* LBA[7:0] */
                SET_FIS_SECTOR(prb->prb_fis, cmd->satacmd_lba_low_lsb);

                /* LBA[15:8] */
                SET_FIS_CYL_LOW(prb->prb_fis, cmd->satacmd_lba_mid_lsb);

                /* LBA[23:16] */
                SET_FIS_CYL_HI(prb->prb_fis, cmd->satacmd_lba_high_lsb);

                /* LBA [31:24] */
                SET_FIS_SECTOR_EXP(prb->prb_fis, cmd->satacmd_lba_low_msb);

                /* LBA [39:32] */
                SET_FIS_CYL_LOW_EXP(prb->prb_fis, cmd->satacmd_lba_mid_msb);

                /* LBA [47:40] */
                SET_FIS_CYL_HI_EXP(prb->prb_fis, cmd->satacmd_lba_high_msb);

                /* Set dev_head */
                SET_FIS_DEV_HEAD(prb->prb_fis, cmd->satacmd_device_reg);

                /* Set the extended sector count and features */
                SET_FIS_SECTOR_COUNT_EXP(prb->prb_fis,
                    cmd->satacmd_sec_count_msb);
                SET_FIS_FEATURES_EXP(prb->prb_fis,
                    cmd->satacmd_features_reg_ext);

                break;

        }

        if (cmd->satacmd_flags.sata_queued) {
                /*
                 * For queued commands, the TAG for the sector count lsb is
                 * generated from current slot number.
                 */
                SET_FIS_SECTOR_COUNT(prb->prb_fis, slot << 3);
        }

        if ((cmd->satacmd_cmd_reg == SATAC_WRITE_FPDMA_QUEUED) ||
            (cmd->satacmd_cmd_reg == SATAC_READ_FPDMA_QUEUED)) {
                si_portp->siport_pending_ncq_count++;
        }

        /* *** now fill the scatter gather list ******* */

        if (is_atapi) { /* It is an ATAPI drive */
                /* atapi command goes into sge0 */
                bcopy(cmd->satacmd_acdb, &prb->prb_sge0, sizeof (si_sge_t));

                /* Now fill sge1 with pointer to external SGT. */
                if (spkt->satapkt_cmd.satacmd_num_dma_cookies) {
                        prb->prb_sge1.sge_addr =
                            si_portp->siport_sgbpool_physaddr +
                            slot * sizeof (si_sgblock_t) * si_dma_sg_number;
                        SET_SGE_LNK(prb->prb_sge1);
                } else {
                        SET_SGE_TRM(prb->prb_sge1);
                }
        } else {
                /* Fill the sge0 */
                if (spkt->satapkt_cmd.satacmd_num_dma_cookies) {
                        prb->prb_sge0.sge_addr =
                            si_portp->siport_sgbpool_physaddr +
                            slot * sizeof (si_sgblock_t) * si_dma_sg_number;
                        SET_SGE_LNK(prb->prb_sge0);

                } else {
                        SET_SGE_TRM(prb->prb_sge0);
                }

                /* sge1 is left empty in non-ATAPI case */
        }

        bzero(&si_portp->siport_sgbpool[slot * si_dma_sg_number],
            sizeof (si_sgblock_t) * si_dma_sg_number);

        ncookies = spkt->satapkt_cmd.satacmd_num_dma_cookies;
        ASSERT(ncookies <= (SGE_LENGTH(si_dma_sg_number)));

        SIDBG_P(SIDBG_COOKIES, si_portp, "total ncookies: %d", ncookies);
        if (ncookies == 0) {
                sgbp = &si_portp->siport_sgbpool[slot * si_dma_sg_number];
                sgtp = &sgbp->sgb_sgt[0];
                sgep = &sgtp->sgt_sge[0];

                /* No cookies. Terminate the chain. */
                SIDBG_P(SIDBG_COOKIES, si_portp, "empty cookies: terminating.",
                    NULL);

                sgep->sge_addr_low = 0;
                sgep->sge_addr_high = 0;
                sgep->sge_data_count = 0;
                SET_SGE_TRM((*sgep));

                goto sgl_fill_done;
        }

        for (i = 0, cookie_index = 0,
            sgbp = &si_portp->siport_sgbpool[slot * si_dma_sg_number];
            i < si_dma_sg_number; i++) {

                sgtp = &sgbp->sgb_sgt[0] + i;

                /* Now fill the first 3 entries of SGT in the loop below. */
                for (j = 0, sgep = &sgtp->sgt_sge[0];
                    ((j < 3) && (cookie_index < ncookies-1));
                    j++, cookie_index++, sgep++)  {
                        ASSERT(cookie_index < ncookies);
                        SIDBG_P(SIDBG_COOKIES, si_portp,
                            "inner loop: cookie_index: %d, ncookies: %d",
                            cookie_index,
                            ncookies);
                        cookie = spkt->satapkt_cmd.
                            satacmd_dma_cookie_list[cookie_index];

                        sgep->sge_addr_low = cookie._dmu._dmac_la[0];
                        sgep->sge_addr_high = cookie._dmu._dmac_la[1];
                        sgep->sge_data_count = (uint32_t)cookie.dmac_size;
                }

                /*
                 * If this happens to be the last cookie, we terminate it here.
                 * Otherwise, we link to next SGT.
                 */

                if (cookie_index == ncookies-1) {
                        /* This is the last cookie. Terminate the chain. */
                        SIDBG_P(SIDBG_COOKIES, si_portp,
                            "filling the last: cookie_index: %d, "
                            "ncookies: %d",
                            cookie_index,
                            ncookies);
                        cookie = spkt->satapkt_cmd.
                            satacmd_dma_cookie_list[cookie_index];

                        sgep->sge_addr_low = cookie._dmu._dmac_la[0];
                        sgep->sge_addr_high = cookie._dmu._dmac_la[1];
                        sgep->sge_data_count = (uint32_t)cookie.dmac_size;
                        SET_SGE_TRM((*sgep));

                        break; /* we break the loop */

                } else {
                        /* This is not the last one. So link it. */
                        SIDBG_P(SIDBG_COOKIES, si_portp,
                            "linking SGT: cookie_index: %d, ncookies: %d",
                            cookie_index,
                            ncookies);
                        sgep->sge_addr = si_portp->siport_sgbpool_physaddr +
                            slot * sizeof (si_sgblock_t) * si_dma_sg_number +
                            (i+1) * sizeof (si_sgt_t);

                        SET_SGE_LNK((*sgep));
                }

        }

        /* *** finished filling the scatter gather list ******* */

sgl_fill_done:
        /* Now remember the sata packet in siport_slot_pkts[]. */
        si_portp->siport_slot_pkts[slot] = spkt;

        /*
         * We are overloading satapkt_hba_driver_private with
         * watched_cycle count.
         */
        spkt->satapkt_hba_driver_private = (void *)(intptr_t)0;

        if (is_atapi) {
                /* program the packet_lenth if it is atapi device. */


#ifdef ATAPI_2nd_PHASE
                /*
                 * Framework needs to calculate the acdb_len based on
                 * identify packet data. This needs to be accomplished
                 * in second phase of the project.
                 */
                ASSERT((cmd->satacmd_acdb_len == 12) ||
                    (cmd->satacmd_acdb_len == 16));
                SIDBG_P(SIDBG_VERBOSE, si_portp, "deliver: acdb_len: %d",
                    cmd->satacmd_acdb_len);

                if (cmd->satacmd_acdb_len == 16) {
                        ddi_put32(si_ctlp->sictl_port_acc_handle,
                            (uint32_t *)PORT_CONTROL_SET(si_ctlp, port),
                            PORT_CONTROL_SET_BITS_PACKET_LEN);
                } else {
                        ddi_put32(si_ctlp->sictl_port_acc_handle,
                            (uint32_t *)PORT_CONTROL_CLEAR(si_ctlp, port),
                            PORT_CONTROL_CLEAR_BITS_PACKET_LEN);
                }

#else /* ATAPI_2nd_PHASE */
                /* hard coding for now to 12 bytes */
                ddi_put32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)PORT_CONTROL_CLEAR(si_ctlp, port),
                    PORT_CONTROL_CLEAR_BITS_PACKET_LEN);
#endif /* ATAPI_2nd_PHASE */
        }


#if SI_DEBUG
        if (si_debug_flags & SIDBG_DUMP_PRB) {
                if (!(is_atapi && (prb->prb_sge0.sge_addr_low == 0))) {
                        /*
                         * Do not dump the atapi Test-Unit-Ready commands.
                         * The sd_media_watch spews too many of these.
                         */
                        int *ptr;
                        si_sge_t *tmpsgep;
                        int j;

                        ptr = (int *)(void *)prb;
                        cmn_err(CE_WARN, "si_deliver_satpkt prb: ");
                        for (j = 0; j < (sizeof (si_prb_t)/4); j++) {
                                cmn_err(CE_WARN, "%x ", ptr[j]);
                        }

                        cmn_err(CE_WARN,
                            "si_deliver_satpkt sgt: low, high, count link");
                        for (j = 0,
                            tmpsgep = (si_sge_t *)
                            &si_portp->siport_sgbpool[slot * si_dma_sg_number];
                            j < (sizeof (si_sgblock_t)/ sizeof (si_sge_t))
                            *si_dma_sg_number;
                            j++, tmpsgep++) {
                                ptr = (int *)(void *)tmpsgep;
                                cmn_err(CE_WARN, "%x %x %x %x",
                                    ptr[0],
                                    ptr[1],
                                    ptr[2],
                                    ptr[3]);
                                if (IS_SGE_TRM_SET((*tmpsgep))) {
                                        break;
                                }

                        }
                }

        }
#endif  /* SI_DEBUG */

        /* Deliver PRB */
        POST_PRB_ADDR(si_ctlp, si_portp, port, slot);

        return (slot);
}

/*
 * Initialize the controller and set up driver data structures.
 *
 * This routine can be called from three separate cases: DDI_ATTACH, PM_LEVEL_D0
 * and DDI_RESUME. The DDI_ATTACH case is different from other two cases; the
 * memory allocation & device signature probing are attempted only during
 * DDI_ATTACH case. In the case of PM_LEVEL_D0 & DDI_RESUME, we are starting
 * from a previously initialized state; so there is no need to allocate memory
 * or to attempt probing the device signatures.
 */
static int
si_initialize_controller(si_ctl_state_t *si_ctlp)
{
        uint32_t port_status;
        uint32_t SStatus;
        uint32_t SControl;
        uint8_t port;
        int loop_count = 0;
        si_port_state_t *si_portp;

        SIDBG_C(SIDBG_INIT, si_ctlp,
            "si3124: si_initialize_controller entered", NULL);

        mutex_enter(&si_ctlp->sictl_mutex);

        /* Remove the Global Reset. */
        ddi_put32(si_ctlp->sictl_global_acc_handle,
            (uint32_t *)GLOBAL_CONTROL_REG(si_ctlp),
            GLOBAL_CONTROL_REG_BITS_CLEAR);

        for (port = 0; port < si_ctlp->sictl_num_ports; port++) {

                if (si_ctlp->sictl_flags & SI_ATTACH) {
                        /*
                         * We allocate the port state only during attach
                         * sequence. We don't want to do it during
                         * suspend/resume sequence.
                         */
                        if (si_alloc_port_state(si_ctlp, port)) {
                                mutex_exit(&si_ctlp->sictl_mutex);
                                return (SI_FAILURE);
                        }
                }

                si_portp = si_ctlp->sictl_ports[port];
                mutex_enter(&si_portp->siport_mutex);
                si_portp->siport_ctlp = si_ctlp;
                si_portp->siport_port_num = port;

                /* Clear Port Reset. */
                ddi_put32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)PORT_CONTROL_SET(si_ctlp, port),
                    PORT_CONTROL_SET_BITS_PORT_RESET);
                ddi_put32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)PORT_CONTROL_CLEAR(si_ctlp, port),
                    PORT_CONTROL_CLEAR_BITS_PORT_RESET);

                /*
                 * Arm the interrupts for: Cmd completion, Cmd error,
                 * Port Ready, PM Change, PhyRdyChange, Commwake,
                 * UnrecFIS, Devxchanged, SDBNotify.
                 */
                ddi_put32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)PORT_INTERRUPT_ENABLE_SET(si_ctlp, port),
                    (INTR_COMMAND_COMPLETE |
                    INTR_COMMAND_ERROR |
                    INTR_PORT_READY |
                    INTR_POWER_CHANGE |
                    INTR_PHYRDY_CHANGE |
                    INTR_COMWAKE_RECEIVED |
                    INTR_UNRECOG_FIS |
                    INTR_DEV_XCHANGED |
                    INTR_SETDEVBITS_NOTIFY));

                /* Now enable the interrupts. */
                si_enable_port_interrupts(si_ctlp, port);

                /*
                 * The following PHY initialization is redundant in
                 * in x86 since the BIOS anyway does this as part of
                 * device enumeration during the power up. But this
                 * is a required step in sparc since there is no BIOS.
                 *
                 * The way to initialize the PHY is to write a 1 and then
                 * a 0 to DET field of SControl register.
                 */

                /*
                 * Fetch the current SControl before writing the
                 * DET part with 1
                 */
                SControl = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)PORT_SCONTROL(si_ctlp, port));
                SCONTROL_SET_DET(SControl, SCONTROL_DET_COMRESET);
                ddi_put32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)(PORT_SCONTROL(si_ctlp, port)),
                    SControl);
#ifndef __lock_lint
                delay(SI_10MS_TICKS); /* give time for COMRESET to percolate */
#endif /* __lock_lint */

                /*
                 * Now fetch the SControl again and rewrite the
                 * DET part with 0
                 */
                SControl = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)PORT_SCONTROL(si_ctlp, port));
                SCONTROL_SET_DET(SControl, SCONTROL_DET_NOACTION);
                ddi_put32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)(PORT_SCONTROL(si_ctlp, port)),
                    SControl);

                /*
                 * PHY may be initialized by now. Check the DET field of
                 * SStatus to determine if there is a device present.
                 *
                 * The DET field is valid only if IPM field indicates that
                 * the interface is in active state.
                 */

                loop_count = 0;
                do {
                        SStatus = ddi_get32(si_ctlp->sictl_port_acc_handle,
                            (uint32_t *)PORT_SSTATUS(si_ctlp, port));

                        if (SSTATUS_GET_IPM(SStatus) !=
                            SSTATUS_IPM_INTERFACE_ACTIVE) {
                                /*
                                 * If the interface is not active, the DET field
                                 * is considered not accurate. So we want to
                                 * continue looping.
                                 */
                                SSTATUS_SET_DET(SStatus,
                                    SSTATUS_DET_NODEV_NOPHY);
                        }

                        if (loop_count++ > SI_POLLRATE_SSTATUS) {
                                /*
                                 * We are effectively timing out after 0.1 sec.
                                 */
                                break;
                        }

                        /* Wait for 10 millisec */
#ifndef __lock_lint
                        delay(SI_10MS_TICKS);
#endif /* __lock_lint */

                } while (SSTATUS_GET_DET(SStatus) !=
                    SSTATUS_DET_DEVPRESENT_PHYONLINE);

                SIDBG_P(SIDBG_POLL_LOOP, si_portp,
                    "si_initialize_controller: 1st loop count: %d, "
                    "SStatus: 0x%x",
                    loop_count,
                    SStatus);

                if ((SSTATUS_GET_IPM(SStatus) !=
                    SSTATUS_IPM_INTERFACE_ACTIVE) ||
                    (SSTATUS_GET_DET(SStatus) !=
                    SSTATUS_DET_DEVPRESENT_PHYONLINE)) {
                        /*
                         * Either the port is not active or there
                         * is no device present.
                         */
                        si_ctlp->sictl_ports[port]->siport_port_type =
                            PORT_TYPE_NODEV;
                        mutex_exit(&si_portp->siport_mutex);
                        continue;
                }

                /* Wait until Port Ready */
                loop_count = 0;
                do {
                        port_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
                            (uint32_t *)PORT_STATUS(si_ctlp, port));

                        if (loop_count++ > SI_POLLRATE_PORTREADY) {
                                /*
                                 * We are effectively timing out after 0.5 sec.
                                 */
                                break;
                        }

                        /* Wait for 10 millisec */
#ifndef __lock_lint
                        delay(SI_10MS_TICKS);
#endif /* __lock_lint */

                } while (!(port_status & PORT_STATUS_BITS_PORT_READY));

                SIDBG_P(SIDBG_POLL_LOOP, si_portp,
                    "si_initialize_controller: 2nd loop count: %d",
                    loop_count);

                if (si_ctlp->sictl_flags & SI_ATTACH) {
                        /*
                         * We want to probe for dev signature only during attach
                         * case. Don't do it during suspend/resume sequence.
                         */
                        if (port_status & PORT_STATUS_BITS_PORT_READY) {
                                mutex_exit(&si_portp->siport_mutex);
                                si_find_dev_signature(si_ctlp, si_portp, port,
                                    PORTMULT_CONTROL_PORT);
                                mutex_enter(&si_portp->siport_mutex);
                        } else {
                                si_ctlp->sictl_ports[port]->siport_port_type =
                                    PORT_TYPE_NODEV;
                        }
                }

                if (si_check_ctl_handles(si_ctlp) != DDI_SUCCESS ||
                    si_check_port_handles(si_portp) != DDI_SUCCESS) {
                        ddi_fm_service_impact(si_ctlp->sictl_devinfop,
                            DDI_SERVICE_LOST);
                        mutex_exit(&si_portp->siport_mutex);
                        mutex_exit(&si_ctlp->sictl_mutex);
                        return (SI_FAILURE);
                }

                mutex_exit(&si_portp->siport_mutex);
        }

        mutex_exit(&si_ctlp->sictl_mutex);
        return (SI_SUCCESS);
}

/*
 * Reverse of si_initialize_controller().
 *
 * WARNING, WARNING: The caller is expected to obtain the sictl_mutex
 * before calling us.
 */
static void
si_deinitialize_controller(si_ctl_state_t *si_ctlp)
{
        int port;

        _NOTE(ASSUMING_PROTECTED(si_ctlp))

        SIDBG_C(SIDBG_INIT, si_ctlp,
            "si3124: si_deinitialize_controller entered", NULL);

        /* disable all the interrupts. */
        si_disable_all_interrupts(si_ctlp);

        if (si_ctlp->sictl_flags & SI_DETACH) {
                /*
                 * We want to dealloc all the memory in detach case.
                 */
                for (port = 0; port < si_ctlp->sictl_num_ports; port++) {
                        si_dealloc_port_state(si_ctlp, port);
                }
        }

}

/*
 * Prepare the port ready for usage.
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static void
si_init_port(si_ctl_state_t *si_ctlp, int port)
{

        SIDBG_C(SIDBG_INIT, si_ctlp,
            "si_init_port entered: port: 0x%x",
            port);

        /* Initialize the port. */
        ddi_put32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)PORT_CONTROL_SET(si_ctlp, port),
            PORT_CONTROL_SET_BITS_PORT_INITIALIZE);

        /*
         * Clear the InterruptNCOR (Interrupt No Clear on Read).
         * This step ensures that a mere reading of slot_status will clear
         * the interrupt; no explicit clearing of interrupt condition
         * will be needed for successful completion of commands.
         */
        ddi_put32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)PORT_CONTROL_CLEAR(si_ctlp, port),
            PORT_CONTROL_CLEAR_BITS_INTR_NCoR);

        /* clear any pending interrupts at this point */
        ddi_put32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_INTERRUPT_STATUS(si_ctlp, port)),
            INTR_MASK);

}


/*
 * Enumerate the devices connected to the port multiplier.
 * Once a device is detected, we call si_find_dev_signature()
 * to find the type of device connected. Even though we are
 * called from within si_find_dev_signature(), there is no
 * recursion possible.
 */
static int
si_enumerate_port_multiplier(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        uint32_t num_dev_ports = 0;
        int pmport;
        uint32_t SControl = 0;
        uint32_t SStatus = 0;
        uint32_t SError = 0;
        int loop_count = 0;

        SIDBG_P(SIDBG_INIT, si_portp,
            "si_enumerate_port_multiplier entered: port: %d",
            port);

        mutex_enter(&si_portp->siport_mutex);

        /* Enable Port Multiplier context switching. */
        ddi_put32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)PORT_CONTROL_SET(si_ctlp, port),
            PORT_CONTROL_SET_BITS_PM_ENABLE);

        /*
         * Read the num dev ports connected.
         * GSCR[2] contains the number of device ports.
         */
        if (si_read_portmult_reg(si_ctlp, si_portp, port, PORTMULT_CONTROL_PORT,
            PSCR_REG2, &num_dev_ports)) {
                mutex_exit(&si_portp->siport_mutex);
                return (SI_FAILURE);
        }
        si_portp->siport_portmult_state.sipm_num_ports = num_dev_ports;

        SIDBG_P(SIDBG_INIT, si_portp,
            "si_enumerate_port_multiplier: ports found: %d",
            num_dev_ports);

        for (pmport = 0; pmport < num_dev_ports-1; pmport++) {
                /*
                 * Enable PHY by writing a 1, then a 0 to SControl
                 * (i.e. PSCR[2]) DET field.
                 */
                if (si_read_portmult_reg(si_ctlp, si_portp, port, pmport,
                    PSCR_REG2, &SControl)) {
                        continue;
                }

                /* First write a 1 to DET field of SControl. */
                SCONTROL_SET_DET(SControl, SCONTROL_DET_COMRESET);
                if (si_write_portmult_reg(si_ctlp, si_portp, port, pmport,
                    PSCR_REG2, SControl)) {
                        continue;
                }
#ifndef __lock_lint
                delay(SI_10MS_TICKS); /* give time for COMRESET to percolate */
#endif /* __lock_lint */

                /* Then write a 0 to the DET field of SControl. */
                SCONTROL_SET_DET(SControl, SCONTROL_DET_NOACTION);
                if (si_write_portmult_reg(si_ctlp, si_portp, port, pmport,
                    PSCR_REG2, SControl)) {
                        continue;
                }

                /* Wait for PHYRDY by polling SStatus (i.e. PSCR[0]). */
                loop_count = 0;
                do {
                        if (si_read_portmult_reg(si_ctlp, si_portp, port,
                            pmport, PSCR_REG0, &SStatus)) {
                                break;
                        }
                        SIDBG_P(SIDBG_POLL_LOOP, si_portp,
                            "looping for PHYRDY: SStatus: %x",
                            SStatus);

                        if (SSTATUS_GET_IPM(SStatus) !=
                            SSTATUS_IPM_INTERFACE_ACTIVE) {
                                /*
                                 * If the interface is not active, the DET field
                                 * is considered not accurate. So we want to
                                 * continue looping.
                                 */
                                SSTATUS_SET_DET(SStatus,
                                    SSTATUS_DET_NODEV_NOPHY);
                        }

                        if (loop_count++ > SI_POLLRATE_SSTATUS) {
                                /*
                                 * We are effectively timing out after 0.1 sec.
                                 */
                                break;
                        }

                        /* Wait for 10 millisec */
#ifndef __lock_lint
                        delay(SI_10MS_TICKS);
#endif /* __lock_lint */

                } while (SSTATUS_GET_DET(SStatus) !=
                    SSTATUS_DET_DEVPRESENT_PHYONLINE);

                SIDBG_P(SIDBG_POLL_LOOP, si_portp,
                    "si_enumerate_port_multiplier: "
                    "loop count: %d, SStatus: 0x%x",
                    loop_count,
                    SStatus);

                if ((SSTATUS_GET_IPM(SStatus) ==
                    SSTATUS_IPM_INTERFACE_ACTIVE) &&
                    (SSTATUS_GET_DET(SStatus) ==
                    SSTATUS_DET_DEVPRESENT_PHYONLINE)) {
                        /* The interface is active and the device is present */
                        SIDBG_P(SIDBG_INIT, si_portp,
                            "Status: %x, device exists",
                            SStatus);
                        /*
                         * Clear error bits in SError register (i.e. PSCR[1]
                         * by writing back error bits.
                         */
                        if (si_read_portmult_reg(si_ctlp, si_portp, port,
                            pmport, PSCR_REG1, &SError)) {
                                continue;
                        }
                        SIDBG_P(SIDBG_INIT, si_portp,
                            "SError bits are: %x", SError);
                        if (si_write_portmult_reg(si_ctlp, si_portp, port,
                            pmport, PSCR_REG1, SError)) {
                                continue;
                        }

                        /* There exists a device. */
                        mutex_exit(&si_portp->siport_mutex);
                        si_find_dev_signature(si_ctlp, si_portp, port, pmport);
                        mutex_enter(&si_portp->siport_mutex);
                }
        }

        mutex_exit(&si_portp->siport_mutex);

        return (SI_SUCCESS);
}


/*
 * Read a port multiplier register.
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static int
si_read_portmult_reg(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port,
        int pmport,
        int regnum,
        uint32_t *regval)
{
        int slot;
        si_prb_t *prb;
        uint32_t *prb_word_ptr;
        int i;
        uint32_t slot_status;
        int loop_count = 0;

        _NOTE(ASSUMING_PROTECTED(si_portp))

        SIDBG_P(SIDBG_ENTRY, si_portp, "si_read_portmult_reg: port: %x,"
            "pmport: %x, regnum: %x",
            port, pmport, regnum);

        slot = si_claim_free_slot(si_ctlp, si_portp, port);
        if (slot == SI_FAILURE) {
                return (SI_FAILURE);
        }

        prb =  &(si_portp->siport_prbpool[slot]);
        bzero((void *)prb, sizeof (si_prb_t));

        /* Now fill the prb. */
        SET_FIS_TYPE(prb->prb_fis, REGISTER_FIS_H2D);
        SET_FIS_PMP(prb->prb_fis, PORTMULT_CONTROL_PORT);
        SET_FIS_CDMDEVCTL(prb->prb_fis, 1);
        SET_FIS_COMMAND(prb->prb_fis, SATAC_READ_PM_REG);

        SET_FIS_DEV_HEAD(prb->prb_fis, pmport);
        SET_FIS_FEATURES(prb->prb_fis, regnum);

        /* no real data transfer is involved. */
        SET_SGE_TRM(prb->prb_sge0);

#if SI_DEBUG
        if (si_debug_flags & SIDBG_DUMP_PRB) {
                int *ptr;
                int j;

                ptr = (int *)(void *)prb;
                cmn_err(CE_WARN, "read_port_mult_reg, prb: ");
                for (j = 0; j < (sizeof (si_prb_t)/4); j++) {
                        cmn_err(CE_WARN, "%x ", ptr[j]);
                }

        }
#endif /* SI_DEBUG */

        /* Deliver PRB */
        POST_PRB_ADDR(si_ctlp, si_portp, port, slot);

        /* Loop till the command is finished. */
        do {
                slot_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)(PORT_SLOT_STATUS(si_ctlp, port)));

                SIDBG_P(SIDBG_POLL_LOOP, si_portp,
                    "looping read_pm slot_status: 0x%x",
                    slot_status);

                if (loop_count++ > SI_POLLRATE_SLOTSTATUS) {
                        /* We are effectively timing out after 0.5 sec. */
                        break;
                }

                /* Wait for 10 millisec */
#ifndef __lock_lint
                delay(SI_10MS_TICKS);
#endif /* __lock_lint */

        } while (slot_status & SI_SLOT_MASK & (0x1 << slot));

        SIDBG_P(SIDBG_POLL_LOOP, si_portp,
            "read_portmult_reg: loop count: %d",
            loop_count);

        CLEAR_BIT(si_portp->siport_pending_tags, slot);

        /* Now inspect the port LRAM for the modified FIS. */
        prb_word_ptr = (uint32_t *)(void *)prb;
        for (i = 0; i < (sizeof (si_prb_t)/4); i++) {
                prb_word_ptr[i] = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)(PORT_LRAM(si_ctlp, port, slot)+i*4));
        }

        if (si_check_ctl_handles(si_ctlp) != DDI_SUCCESS ||
            si_check_port_handles(si_portp) != DDI_SUCCESS) {
                ddi_fm_service_impact(si_ctlp->sictl_devinfop,
                    DDI_SERVICE_UNAFFECTED);
                return (SI_FAILURE);
        }

        if (((GET_FIS_COMMAND(prb->prb_fis) & 0x1) != 0) ||
            (GET_FIS_FEATURES(prb->prb_fis) != 0)) {
                /* command failed. */
                return (SI_FAILURE);
        }

        /* command succeeded. */
        *regval = (GET_FIS_SECTOR_COUNT(prb->prb_fis) & 0xff) |
            ((GET_FIS_SECTOR(prb->prb_fis) << 8)  & 0xff00) |
            ((GET_FIS_CYL_LOW(prb->prb_fis) << 16)  & 0xff0000) |
            ((GET_FIS_CYL_HI(prb->prb_fis) << 24)  & 0xff000000);

        return (SI_SUCCESS);
}

/*
 * Write a port multiplier register.
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static int
si_write_portmult_reg(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port,
        int pmport,
        int regnum,
        uint32_t regval)
{
        int slot;
        si_prb_t *prb;
        uint32_t *prb_word_ptr;
        uint32_t slot_status;
        int i;
        int loop_count = 0;

        _NOTE(ASSUMING_PROTECTED(si_portp))

        SIDBG_P(SIDBG_ENTRY, si_portp,
            "si_write_portmult_reg: port: %x, pmport: %x,"
            "regnum: %x, regval: %x",
            port, pmport, regnum, regval);

        slot = si_claim_free_slot(si_ctlp, si_portp, port);
        if (slot == SI_FAILURE) {
                return (SI_FAILURE);
        }

        prb =  &(si_portp->siport_prbpool[slot]);
        bzero((void *)prb, sizeof (si_prb_t));

        /* Now fill the prb. */
        SET_FIS_TYPE(prb->prb_fis, REGISTER_FIS_H2D);
        SET_FIS_PMP(prb->prb_fis, PORTMULT_CONTROL_PORT);
        SET_FIS_CDMDEVCTL(prb->prb_fis, 1);

        SET_FIS_COMMAND(prb->prb_fis, SATAC_WRITE_PM_REG);
        SET_FIS_DEV_HEAD(prb->prb_fis, pmport);
        SET_FIS_FEATURES(prb->prb_fis, regnum);

        SET_FIS_SECTOR_COUNT(prb->prb_fis, regval & 0xff);
        SET_FIS_SECTOR(prb->prb_fis, (regval >> 8) & 0xff);
        SET_FIS_CYL_LOW(prb->prb_fis, (regval >> 16) & 0xff);
        SET_FIS_CYL_HI(prb->prb_fis, (regval >> 24)  & 0xff);

        /* no real data transfer is involved. */
        SET_SGE_TRM(prb->prb_sge0);

#if SI_DEBUG
        if (si_debug_flags & SIDBG_DUMP_PRB) {
                int *ptr;
                int j;

                ptr = (int *)(void *)prb;
                cmn_err(CE_WARN, "read_port_mult_reg, prb: ");
                for (j = 0; j < (sizeof (si_prb_t)/4); j++) {
                        cmn_err(CE_WARN, "%x ", ptr[j]);
                }

        }
#endif /* SI_DEBUG */

        /* Deliver PRB */
        POST_PRB_ADDR(si_ctlp, si_portp, port, slot);

        /* Loop till the command is finished. */
        do {
                slot_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)(PORT_SLOT_STATUS(si_ctlp, port)));

                SIDBG_P(SIDBG_POLL_LOOP, si_portp,
                    "looping write_pmp slot_status: 0x%x",
                    slot_status);

                if (loop_count++ > SI_POLLRATE_SLOTSTATUS) {
                        /* We are effectively timing out after 0.5 sec. */
                        break;
                }

                /* Wait for 10 millisec */
#ifndef __lock_lint
                delay(SI_10MS_TICKS);
#endif /* __lock_lint */

        } while (slot_status & SI_SLOT_MASK & (0x1 << slot));

        SIDBG_P(SIDBG_POLL_LOOP, si_portp,
            "write_portmult_reg: loop count: %d",
            loop_count);

        CLEAR_BIT(si_portp->siport_pending_tags, slot);

        /* Now inspect the port LRAM for the modified FIS. */
        prb_word_ptr = (uint32_t *)(void *)prb;
        for (i = 0; i < (sizeof (si_prb_t)/4); i++) {
                prb_word_ptr[i] = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)(PORT_LRAM(si_ctlp, port, slot)+i*4));
        }

        if (si_check_ctl_handles(si_ctlp) != DDI_SUCCESS ||
            si_check_port_handles(si_portp) != DDI_SUCCESS) {
                ddi_fm_service_impact(si_ctlp->sictl_devinfop,
                    DDI_SERVICE_UNAFFECTED);
                return (SI_FAILURE);
        }

        if (((GET_FIS_COMMAND(prb->prb_fis) & 0x1) != 0) ||
            (GET_FIS_FEATURES(prb->prb_fis) != 0)) {
                /* command failed */
                return (SI_FAILURE);
        }

        /* command succeeded */
        return (SI_SUCCESS);
}


/*
 * Set the auto sense data for ATAPI devices.
 *
 * Note: Currently the sense data is simulated; this code will be enhanced
 * in second phase to fetch the real sense data from the atapi device.
 */
static void
si_set_sense_data(sata_pkt_t *satapkt, int reason)
{
        struct scsi_extended_sense *sense;

        sense = (struct scsi_extended_sense *)
            satapkt->satapkt_cmd.satacmd_rqsense;
        bzero(sense, sizeof (struct scsi_extended_sense));
        sense->es_valid = 1;            /* Valid sense */
        sense->es_class = 7;            /* Response code 0x70 - current err */
        sense->es_key = 0;
        sense->es_info_1 = 0;
        sense->es_info_2 = 0;
        sense->es_info_3 = 0;
        sense->es_info_4 = 0;
        sense->es_add_len = 6;          /* Additional length */
        sense->es_cmd_info[0] = 0;
        sense->es_cmd_info[1] = 0;
        sense->es_cmd_info[2] = 0;
        sense->es_cmd_info[3] = 0;
        sense->es_add_code = 0;
        sense->es_qual_code = 0;

        if ((reason == SATA_PKT_DEV_ERROR) || (reason == SATA_PKT_TIMEOUT)) {
                sense->es_key = KEY_HARDWARE_ERROR;
        }
}


/*
 * Interrupt service handler. We loop through each of the ports to find
 * if the interrupt belongs to any of them.
 *
 * Bulk of the interrupt handling is actually done out of subroutines
 * like si_intr_command_complete() etc.
 */
/*ARGSUSED*/
static uint_t
si_intr(caddr_t arg1, caddr_t arg2)
{
        si_ctl_state_t *si_ctlp = (si_ctl_state_t *)(void *)arg1;
        si_port_state_t *si_portp;
        uint32_t global_intr_status;
        uint32_t mask, port_intr_status;
        int port;

        global_intr_status = ddi_get32(si_ctlp->sictl_global_acc_handle,
            (uint32_t *)GLOBAL_INTERRUPT_STATUS(si_ctlp));

        SIDBG_C(SIDBG_INTR, si_ctlp,
            "si_intr: global_int_status: 0x%x",
            global_intr_status);

        if (si_check_acc_handle(si_ctlp->sictl_global_acc_handle) !=
            DDI_SUCCESS) {
                ddi_fm_service_impact(si_ctlp->sictl_devinfop,
                    DDI_SERVICE_UNAFFECTED);
                return (DDI_INTR_UNCLAIMED);
        }

        if (!(global_intr_status & SI31xx_INTR_PORT_MASK)) {
                /* Sorry, the interrupt is not ours. */
                return (DDI_INTR_UNCLAIMED);
        }

        /* Loop for all the ports. */
        for (port = 0; port < si_ctlp->sictl_num_ports; port++) {

                mask = 0x1 << port;
                if (!(global_intr_status & mask)) {
                        continue;
                }

                mutex_enter(&si_ctlp->sictl_mutex);
                si_portp = si_ctlp->sictl_ports[port];
                mutex_exit(&si_ctlp->sictl_mutex);

                port_intr_status = ddi_get32(si_ctlp->sictl_global_acc_handle,
                    (uint32_t *)PORT_INTERRUPT_STATUS(si_ctlp, port));

                SIDBG_P(SIDBG_VERBOSE, si_portp,
                    "s_intr: port_intr_status: 0x%x, port: %x",
                    port_intr_status,
                    port);

                if (port_intr_status & INTR_COMMAND_COMPLETE) {
                        (void) si_intr_command_complete(si_ctlp, si_portp,
                            port);

                        mutex_enter(&si_portp->siport_mutex);
                        if (si_check_ctl_handles(si_ctlp) != DDI_SUCCESS ||
                            si_check_port_handles(si_portp) != DDI_SUCCESS) {
                                ddi_fm_service_impact(si_ctlp->sictl_devinfop,
                                    DDI_SERVICE_UNAFFECTED);
                                si_schedule_port_initialize(si_ctlp, si_portp,
                                    port);
                        }
                        mutex_exit(&si_portp->siport_mutex);
                } else {
                        /* Clear the interrupts */
                        ddi_put32(si_ctlp->sictl_port_acc_handle,
                            (uint32_t *)(PORT_INTERRUPT_STATUS(si_ctlp, port)),
                            port_intr_status & INTR_MASK);
                }

                /*
                 * Note that we did not clear the interrupt for command
                 * completion interrupt. Reading of slot_status takes care
                 * of clearing the interrupt for command completion case.
                 */

                if (port_intr_status & INTR_COMMAND_ERROR) {
                        si_schedule_intr_command_error(si_ctlp, si_portp, port);
                }

                if (port_intr_status & INTR_PORT_READY) {
                        (void) si_intr_port_ready(si_ctlp, si_portp, port);
                }

                if (port_intr_status & INTR_POWER_CHANGE) {
                        (void) si_intr_pwr_change(si_ctlp, si_portp, port);
                }

                if (port_intr_status & INTR_PHYRDY_CHANGE) {
                        (void) si_intr_phy_ready_change(si_ctlp, si_portp,
                            port);
                }

                if (port_intr_status & INTR_COMWAKE_RECEIVED) {
                        (void) si_intr_comwake_rcvd(si_ctlp, si_portp,
                            port);
                }

                if (port_intr_status & INTR_UNRECOG_FIS) {
                        (void) si_intr_unrecognised_fis(si_ctlp, si_portp,
                            port);
                }

                if (port_intr_status & INTR_DEV_XCHANGED) {
                        (void) si_intr_dev_xchanged(si_ctlp, si_portp, port);
                }

                if (port_intr_status & INTR_8B10B_DECODE_ERROR) {
                        (void) si_intr_decode_err_threshold(si_ctlp, si_portp,
                            port);
                }

                if (port_intr_status & INTR_CRC_ERROR) {
                        (void) si_intr_crc_err_threshold(si_ctlp, si_portp,
                            port);
                }

                if (port_intr_status & INTR_HANDSHAKE_ERROR) {
                        (void) si_intr_handshake_err_threshold(si_ctlp,
                            si_portp, port);
                }

                if (port_intr_status & INTR_SETDEVBITS_NOTIFY) {
                        (void) si_intr_set_devbits_notify(si_ctlp, si_portp,
                            port);
                }
        }

        return (DDI_INTR_CLAIMED);
}

/*
 * Interrupt which indicates that one or more commands have successfully
 * completed.
 *
 * Since we disabled W1C (write-one-to-clear) previously, mere reading
 * of slot_status register clears the interrupt. There is no need to
 * explicitly clear the interrupt.
 */
static int
si_intr_command_complete(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{

        uint32_t slot_status;
        uint32_t finished_tags;
        int finished_slot;
        sata_pkt_t *satapkt;

        SIDBG_P(SIDBG_INTR, si_portp,
            "si_intr_command_complete enter", NULL);

        mutex_enter(&si_portp->siport_mutex);

        slot_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_SLOT_STATUS(si_ctlp, port)));

        if (!si_portp->siport_pending_tags) {
                /*
                 * Spurious interrupt. Nothing to be done.
                 * The interrupt was cleared when slot_status was read.
                 */
                mutex_exit(&si_portp->siport_mutex);
                return (SI_SUCCESS);
        }

        SIDBG_P(SIDBG_VERBOSE, si_portp, "si3124: si_intr_command_complete: "
            "pending_tags: %x, slot_status: %x",
            si_portp->siport_pending_tags,
            slot_status);

        finished_tags =  si_portp->siport_pending_tags &
            ~slot_status & SI_SLOT_MASK;
        while (finished_tags) {

                finished_slot = ddi_ffs(finished_tags) - 1;
                if (finished_slot == -1) {
                        break;
                }

                satapkt = si_portp->siport_slot_pkts[finished_slot];

                if (satapkt->satapkt_cmd.satacmd_flags.sata_special_regs) {
                        si_copy_out_regs(&satapkt->satapkt_cmd, si_ctlp, port,
                            finished_slot);
                }

                CLEAR_BIT(si_portp->siport_pending_tags, finished_slot);
                CLEAR_BIT(finished_tags, finished_slot);
                SENDUP_PACKET(si_portp, satapkt, SATA_PKT_COMPLETED);
        }

        SIDBG_P(SIDBG_PKTCOMP, si_portp,
            "command_complete done: pend_tags: 0x%x, slot_status: 0x%x",
            si_portp->siport_pending_tags,
            slot_status);

        /*
         * tidbit: no need to clear the interrupt since reading of
         * slot_status automatically clears the interrupt in the case
         * of a successful command completion.
         */

        mutex_exit(&si_portp->siport_mutex);

        return (SI_SUCCESS);
}

/*
 * Schedule a call to si_intr_command_error using a timeout to get it done
 * off the interrupt thread.
 */
static void
si_schedule_intr_command_error(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        si_event_arg_t *args;

        mutex_enter(&si_portp->siport_mutex);

        args = si_portp->siport_event_args;
        if (args->siea_ctlp != NULL) {
                cmn_err(CE_WARN, "si_schedule_intr_command_error: "
                    "args->si_ctlp != NULL");
                mutex_exit(&si_portp->siport_mutex);
                return;
        }

        args->siea_ctlp = si_ctlp;
        args->siea_port = port;

        (void) timeout(si_do_intr_command_error, si_portp, 1);

        mutex_exit(&si_portp->siport_mutex);
}

/*
 * Called from timeout()
 * Unpack the arguments and call si_intr_command_error()
 */
static void
si_do_intr_command_error(void *arg)
{
        si_event_arg_t *args;
        si_ctl_state_t *si_ctlp;
        si_port_state_t *si_portp;
        int port;

        si_portp = arg;
        mutex_enter(&si_portp->siport_mutex);

        args = si_portp->siport_event_args;
        si_ctlp = args->siea_ctlp;
        port = args->siea_port;
        args->siea_ctlp = NULL; /* mark siport_event_args as free */

        mutex_exit(&si_portp->siport_mutex);
        (void) si_intr_command_error(si_ctlp, si_portp, port);
}

/*
 * Interrupt which indicates that a command did not complete successfully.
 *
 * The port halts whenever a command error interrupt is received.
 * The only way to restart it is to reset or reinitialize the port
 * but such an operation throws away all the pending commands on
 * the port.
 *
 * We reset the device and mop the commands on the port.
 */
static int
si_intr_command_error(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        uint32_t command_error, slot_status;
        uint32_t failed_tags;

        command_error = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_COMMAND_ERROR(si_ctlp, port)));

        SIDBG_P(SIDBG_ERRS, si_portp,
            "si_intr_command_error: command_error: 0x%x",
            command_error);

        mutex_enter(&si_portp->siport_mutex);

        /*
         * Remember the slot_status since any of the recovery handler
         * can blow it away with reset operation.
         */
        slot_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_SLOT_STATUS(si_ctlp, port)));

        si_log_error_message(si_ctlp, port, command_error);

        switch (command_error) {

        case CMD_ERR_DEVICEERRROR:
                si_error_recovery_DEVICEERROR(si_ctlp, si_portp, port);
                break;

        case CMD_ERR_SDBERROR:
                si_fm_ereport(si_ctlp, DDI_FM_DEVICE_INTERN_CORR, "SBD error");
                si_error_recovery_SDBERROR(si_ctlp, si_portp, port);
                ddi_fm_service_impact(si_ctlp->sictl_devinfop,
                    DDI_SERVICE_UNAFFECTED);
                break;

        case CMD_ERR_DATAFISERROR:
                si_fm_ereport(si_ctlp, DDI_FM_DEVICE_INTERN_CORR,
                    "Data FIS error");
                si_error_recovery_DATAFISERROR(si_ctlp, si_portp, port);
                ddi_fm_service_impact(si_ctlp->sictl_devinfop,
                    DDI_SERVICE_UNAFFECTED);
                break;

        case CMD_ERR_SENDFISERROR:
                si_fm_ereport(si_ctlp, DDI_FM_DEVICE_INTERN_CORR,
                    "Send FIS error");
                si_error_recovery_SENDFISERROR(si_ctlp, si_portp, port);
                ddi_fm_service_impact(si_ctlp->sictl_devinfop,
                    DDI_SERVICE_UNAFFECTED);
                break;

        default:
                si_fm_ereport(si_ctlp, DDI_FM_DEVICE_INTERN_CORR,
                    "Unknown error");
                si_error_recovery_default(si_ctlp, si_portp, port);
                ddi_fm_service_impact(si_ctlp->sictl_devinfop,
                    DDI_SERVICE_UNAFFECTED);
                break;

        }

        /*
         * Compute the failed_tags by adding up the error tags.
         *
         * The siport_err_tags_SDBERROR and siport_err_tags_nonSDBERROR
         * were filled in by the si_error_recovery_* routines.
         */
        failed_tags = si_portp->siport_pending_tags &
            (si_portp->siport_err_tags_SDBERROR |
            si_portp->siport_err_tags_nonSDBERROR);

        SIDBG_P(SIDBG_ERRS, si_portp, "si_intr_command_error: "
            "err_tags_SDBERROR: 0x%x, "
            "err_tags_nonSDBERRROR: 0x%x, "
            "failed_tags: 0x%x",
            si_portp->siport_err_tags_SDBERROR,
            si_portp->siport_err_tags_nonSDBERROR,
            failed_tags);

        SIDBG_P(SIDBG_ERRS, si_portp,
            "si3124: si_intr_command_error: "
            "slot_status:0x%x, pending_tags: 0x%x",
            slot_status,
            si_portp->siport_pending_tags);

        si_portp->mopping_in_progress++;

        si_mop_commands(si_ctlp,
            si_portp,
            port,
            slot_status,
            failed_tags,
            0,  /* timedout_tags */
            0,  /* aborting_tags */
            0); /* reset_tags */

        ASSERT(si_portp->siport_pending_tags == 0);

        si_portp->siport_err_tags_SDBERROR = 0;
        si_portp->siport_err_tags_nonSDBERROR = 0;

        mutex_exit(&si_portp->siport_mutex);

        return (SI_SUCCESS);
}

/*
 * There is a subtle difference between errors on a normal port and
 * a port-mult port. When an error happens on a normal port, the port
 * is halted effectively until the port is reset or initialized.
 * However, in port-mult port errors, port does not get halted since
 * other non-error devices behind the port multiplier can still
 * continue to operate. So we wait till all the commands are drained
 * instead of resetting it right away.
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static void
si_recover_portmult_errors(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        uint32_t command_error, slot_status, port_status;
        int failed_slot;
        int loop_count = 0;

        _NOTE(ASSUMING_PROTECTED(si_portp))

        SIDBG_P(SIDBG_ERRS, si_portp,
            "si_recover_portmult_errors: port: 0x%x",
            port);

        /* Resume the port */
        ddi_put32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)PORT_CONTROL_SET(si_ctlp, port),
            PORT_CONTROL_SET_BITS_RESUME);

        port_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)PORT_STATUS(si_ctlp, port));

        failed_slot = (port_status >> 16) & SI_NUM_SLOTS;
        command_error = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_COMMAND_ERROR(si_ctlp, port)));

        if (command_error ==  CMD_ERR_SDBERROR) {
                si_portp->siport_err_tags_SDBERROR |= (0x1 << failed_slot);
        } else {
                si_portp->siport_err_tags_nonSDBERROR |= (0x1 << failed_slot);
        }

        /* Now we drain the pending commands. */
        do {
                slot_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)(PORT_SLOT_STATUS(si_ctlp, port)));

                /*
                 * Since we have not yet returned DDI_INTR_CLAIMED,
                 * our interrupt handler is guaranteed not to be called again.
                 * So we need to check IS_ATTENTION_RAISED() for further
                 * decisions.
                 *
                 * This is a too big a delay for an interrupt context.
                 * But this is supposed to be a rare condition.
                 */

                if (IS_ATTENTION_RAISED(slot_status)) {
                        /* Resume again */
                        ddi_put32(si_ctlp->sictl_port_acc_handle,
                            (uint32_t *)PORT_CONTROL_SET(si_ctlp, port),
                            PORT_CONTROL_SET_BITS_RESUME);

                        port_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
                            (uint32_t *)PORT_STATUS(si_ctlp, port));
                        failed_slot = (port_status >> 16) & SI_NUM_SLOTS;
                        command_error = ddi_get32(
                            si_ctlp->sictl_port_acc_handle,
                            (uint32_t *)(PORT_COMMAND_ERROR(si_ctlp,
                            port)));
                        if (command_error ==  CMD_ERR_SDBERROR) {
                                si_portp->siport_err_tags_SDBERROR |=
                                    (0x1 << failed_slot);
                        } else {
                                si_portp->siport_err_tags_nonSDBERROR |=
                                    (0x1 << failed_slot);
                        }
                }

                if (loop_count++ > SI_POLLRATE_RECOVERPORTMULT) {
                        /* We are effectively timing out after 10 sec. */
                        break;
                }

                /* Wait for 10 millisec */
#ifndef __lock_lint
                delay(SI_10MS_TICKS);
#endif /* __lock_lint */

        } while (slot_status & SI_SLOT_MASK);

        /*
         * The above loop can be improved for 3132 since we could obtain the
         * Port Multiplier Context of the device in error. Then we could
         * do a better job in filtering out commands for the device in error.
         * The loop could finish much earlier with such a logic.
         */

        /* Clear the RESUME bit. */
        ddi_put32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)PORT_CONTROL_CLEAR(si_ctlp, port),
            PORT_CONTROL_CLEAR_BITS_RESUME);

}

/*
 * If we are connected to port multiplier, drain the non-failed devices.
 * Otherwise, we initialize the port (which effectively fails all the
 * pending commands in the hope that sd would retry them later).
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static void
si_error_recovery_DEVICEERROR(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        uint32_t port_status;
        int failed_slot;

        _NOTE(ASSUMING_PROTECTED(si_portp))

        SIDBG_P(SIDBG_ERRS, si_portp,
            "si_error_recovery_DEVICEERROR: port: 0x%x",
            port);

        if (si_portp->siport_port_type == PORT_TYPE_MULTIPLIER) {
                si_recover_portmult_errors(si_ctlp, si_portp, port);
        } else {
                port_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)PORT_STATUS(si_ctlp, port));
                failed_slot = (port_status >> 16) & SI_NUM_SLOTS;
                si_portp->siport_err_tags_nonSDBERROR |= (0x1 << failed_slot);
        }

        /* In either case (port-mult or not), we reinitialize the port. */
        (void) si_initialize_port_wait_till_ready(si_ctlp, port);
}

/*
 * Handle exactly like DEVICEERROR. Remember the tags with SDBERROR
 * to perform read_log_ext on them later. SDBERROR means that the
 * error was for an NCQ command.
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static void
si_error_recovery_SDBERROR(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        uint32_t port_status;
        int failed_slot;

        _NOTE(ASSUMING_PROTECTED(si_portp))

        SIDBG_P(SIDBG_ERRS, si_portp,
            "si3124: si_error_recovery_SDBERROR: port: 0x%x",
            port);

        if (si_portp->siport_port_type == PORT_TYPE_MULTIPLIER) {
                si_recover_portmult_errors(si_ctlp, si_portp, port);
        } else {
                port_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)PORT_STATUS(si_ctlp, port));
                failed_slot = (port_status >> 16) & SI_NUM_SLOTS;
                si_portp->siport_err_tags_SDBERROR |= (0x1 << failed_slot);
        }

        /* In either case (port-mult or not), we reinitialize the port. */
        (void) si_initialize_port_wait_till_ready(si_ctlp, port);
}

/*
 * Handle exactly like DEVICEERROR except resetting the port if there was
 * an NCQ command on the port.
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static void
si_error_recovery_DATAFISERROR(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        uint32_t port_status;
        int failed_slot;

        _NOTE(ASSUMING_PROTECTED(si_portp))

        SIDBG_P(SIDBG_ERRS, si_portp,
            "si3124: si_error_recovery_DATAFISERROR: port: 0x%x",
            port);

        /* reset device if we were waiting for any ncq commands. */
        if (si_portp->siport_pending_ncq_count) {
                port_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)PORT_STATUS(si_ctlp, port));
                failed_slot = (port_status >> 16) & SI_NUM_SLOTS;
                si_portp->siport_err_tags_nonSDBERROR |= (0x1 << failed_slot);
                (void) si_reset_dport_wait_till_ready(si_ctlp, si_portp, port,
                    SI_DEVICE_RESET);
                return;
        }

        /*
         * If we don't have any ncq commands pending, the rest of
         * the process is similar to the one for DEVICEERROR.
         */
        si_error_recovery_DEVICEERROR(si_ctlp, si_portp, port);
}

/*
 * We handle just like DEVICERROR except that we reset the device instead
 * of initializing the port.
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static void
si_error_recovery_SENDFISERROR(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        uint32_t port_status;
        int failed_slot;

        _NOTE(ASSUMING_PROTECTED(si_portp))

        SIDBG_P(SIDBG_ERRS, si_portp,
            "si3124: si_error_recovery_SENDFISERROR: port: 0x%x",
            port);

        if (si_portp->siport_port_type == PORT_TYPE_MULTIPLIER) {
                si_recover_portmult_errors(si_ctlp, si_portp, port);
        } else {
                port_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)PORT_STATUS(si_ctlp, port));
                failed_slot = (port_status >> 16) & SI_NUM_SLOTS;
                si_portp->siport_err_tags_nonSDBERROR |= (0x1 << failed_slot);
                (void) si_reset_dport_wait_till_ready(si_ctlp, si_portp, port,
                    SI_DEVICE_RESET);
        }
}

/*
 * The default behavior for all other errors is to reset the device.
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static void
si_error_recovery_default(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        uint32_t port_status;
        int failed_slot;

        _NOTE(ASSUMING_PROTECTED(si_portp))

        SIDBG_P(SIDBG_ERRS, si_portp,
            "si3124: si_error_recovery_default: port: 0x%x",
            port);

        port_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)PORT_STATUS(si_ctlp, port));
        failed_slot = (port_status >> 16) & SI_NUM_SLOTS;
        si_portp->siport_err_tags_nonSDBERROR |= (0x1 << failed_slot);

        (void) si_reset_dport_wait_till_ready(si_ctlp, si_portp, port,
            SI_DEVICE_RESET);
}

/*
 * Read Log Ext with PAGE 10 to retrieve the error for an NCQ command.
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static uint8_t
si_read_log_ext(si_ctl_state_t *si_ctlp, si_port_state_t *si_portp, int port)
{
        int slot;
        si_prb_t *prb;
        int i;
        uint32_t slot_status;
        int loop_count = 0;
        uint32_t *prb_word_ptr;
        uint8_t error;

        _NOTE(ASSUMING_PROTECTED(si_portp))

        SIDBG_P(SIDBG_ERRS, si_portp,
            "si_read_log_ext: port: %x", port);

        slot = si_claim_free_slot(si_ctlp, si_portp, port);
        if (slot == SI_FAILURE) {
                return (0);
        }

        prb =  &(si_portp->siport_prbpool[slot]);
        bzero((void *)prb, sizeof (si_prb_t));

        /* Now fill the prb */
        SET_FIS_TYPE(prb->prb_fis, REGISTER_FIS_H2D);
        SET_FIS_PMP(prb->prb_fis, PORTMULT_CONTROL_PORT);
        SET_FIS_CDMDEVCTL(prb->prb_fis, 1);
        SET_FIS_COMMAND(prb->prb_fis, SATAC_READ_LOG_EXT);
        SET_FIS_SECTOR(prb->prb_fis, SATA_LOG_PAGE_10);

        /* no real data transfer is involved */
        SET_SGE_TRM(prb->prb_sge0);

#if SI_DEBUG
        if (si_debug_flags & SIDBG_DUMP_PRB) {
                int *ptr;
                int j;

                ptr = (int *)(void *)prb;
                cmn_err(CE_WARN, "read_port_mult_reg, prb: ");
                for (j = 0; j < (sizeof (si_prb_t)/4); j++) {
                        cmn_err(CE_WARN, "%x ", ptr[j]);
                }

        }
#endif /* SI_DEBUG */

        /* Deliver PRB */
        POST_PRB_ADDR(si_ctlp, si_portp, port, slot);

        /* Loop till the command is finished. */
        do {
                slot_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)(PORT_SLOT_STATUS(si_ctlp, port)));

                SIDBG_P(SIDBG_POLL_LOOP, si_portp,
                    "looping read_log_ext slot_status: 0x%x",
                    slot_status);

                if (loop_count++ > SI_POLLRATE_SLOTSTATUS) {
                        /* We are effectively timing out after 0.5 sec. */
                        break;
                }

                /* Wait for 10 millisec */
#ifndef __lock_lint
                delay(SI_10MS_TICKS);
#endif /* __lock_lint */

        } while (slot_status & SI_SLOT_MASK & (0x1 << slot));

        if (slot_status & SI_SLOT_MASK & (0x1 << slot)) {
                /*
                 * If we fail with the READ LOG EXT command, we need to
                 * initialize the port to clear the slot_status register.
                 * We don't need to worry about any other valid commands
                 * being thrown away because we are already in recovery
                 * mode and READ LOG EXT is the only pending command.
                 */
                (void) si_initialize_port_wait_till_ready(si_ctlp, port);
        }

        SIDBG_P(SIDBG_POLL_LOOP, si_portp,
            "read_portmult_reg: loop count: %d",
            loop_count);

        /*
         * The LRAM contains the the modified FIS.
         * Read the modified FIS to obtain the Error.
         */
        prb_word_ptr = (uint32_t *)(void *)prb;
        for (i = 0; i < (sizeof (si_prb_t)/4); i++) {
                prb_word_ptr[i] = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)(PORT_LRAM(si_ctlp, port, slot)+i*4));
        }

        if (si_check_ctl_handles(si_ctlp) != DDI_SUCCESS ||
            si_check_port_handles(si_portp) != DDI_SUCCESS) {
                ddi_fm_service_impact(si_ctlp->sictl_devinfop,
                    DDI_SERVICE_UNAFFECTED);
        }

        error = GET_FIS_FEATURES(prb->prb_fis);

        CLEAR_BIT(si_portp->siport_pending_tags, slot);

        return (error);

}

/*
 * Dump the error message to the log.
 */
static void
si_log_error_message(si_ctl_state_t *si_ctlp, int port, uint32_t command_error)
{
#if SI_DEBUG
#ifndef __lock_lint
        _NOTE(ARGUNUSED(si_ctlp))
        _NOTE(ARGUNUSED(port))
#endif  /* __lock_lint */

        char *errstr;
        si_port_state_t *si_portp = si_ctlp->sictl_ports[port];

        switch (command_error) {

        case CMD_ERR_DEVICEERRROR:
                errstr = "Standard Error: Error bit set in register - device"
                    " to host FIS";
                break;

        case CMD_ERR_SDBERROR:
                errstr = "NCQ Error: Error bit set in register - device"
                    " to host FIS";
                break;

        case CMD_ERR_DATAFISERROR:
                errstr = "Error in data FIS not detected by device";
                break;

        case CMD_ERR_SENDFISERROR:
                errstr = "Initial command FIS transmission failed";
                break;

        case CMD_ERR_INCONSISTENTSTATE:
                errstr = "Inconsistency in protocol";
                break;

        case CMD_ERR_DIRECTIONERROR:
                errstr = "DMA direction flag does not match the command";
                break;

        case CMD_ERR_UNDERRUNERROR:
                errstr = "Run out of scatter gather entries while writing data";
                break;

        case CMD_ERR_OVERRUNERROR:
                errstr = "Run out of scatter gather entries while reading data";
                break;

        case CMD_ERR_PACKETPROTOCOLERROR:
                errstr = "Packet protocol error";
                break;

        case CMD_ERR_PLDSGTERRORBOUNDARY:
                errstr = "Scatter/gather table not on quadword boundary";
                break;

        case CMD_ERR_PLDSGTERRORTARETABORT:
                errstr = "PCI(X) Target abort while fetching scatter/gather"
                    " table";
                break;

        case CMD_ERR_PLDSGTERRORMASTERABORT:
                errstr = "PCI(X) Master abort while fetching scatter/gather"
                    " table";
                break;

        case CMD_ERR_PLDSGTERRORPCIERR:
                errstr = "PCI(X) parity error while fetching scatter/gather"
                    " table";
                break;

        case CMD_ERR_PLDCMDERRORBOUNDARY:
                errstr = "PRB not on quadword boundary";
                break;

        case CMD_ERR_PLDCMDERRORTARGETABORT:
                errstr = "PCI(X) Target abort while fetching PRB";
                break;

        case CMD_ERR_PLDCMDERRORMASTERABORT:
                errstr = "PCI(X) Master abort while fetching PRB";
                break;

        case CMD_ERR_PLDCMDERORPCIERR:
                errstr = "PCI(X) parity error while fetching PRB";
                break;

        case CMD_ERR_PSDERRORTARGETABORT:
                errstr = "PCI(X) Target abort during data transfer";
                break;

        case CMD_ERR_PSDERRORMASTERABORT:
                errstr = "PCI(X) Master abort during data transfer";
                break;

        case CMD_ERR_PSDERRORPCIERR:
                errstr = "PCI(X) parity error during data transfer";
                break;

        case CMD_ERR_SENDSERVICEERROR:
                errstr = "FIS received while sending service FIS in"
                    " legacy queuing operation";
                break;

        default:
                errstr = "Unknown Error";
                break;

        }

        SIDBG_P(SIDBG_ERRS, si_portp,
            "command error: error: %s",
            errstr);
#else
#ifndef __lock_lint
        _NOTE(ARGUNUSED(si_ctlp))
        _NOTE(ARGUNUSED(port))
        _NOTE(ARGUNUSED(command_error))
#endif  /* __lock_lint */

#endif  /* SI_DEBUG */
}


/*
 * Interrupt which indicates that the Port Ready state has changed
 * from zero to one.
 *
 * We are not interested in this interrupt; we just log a debug message.
 */
/*ARGSUSED*/
static int
si_intr_port_ready(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        SIDBG_P(SIDBG_INTR, si_portp, "si_intr_ready", NULL);
        return (SI_SUCCESS);
}

/*
 * Interrupt which indicates that the port power management state
 * has been modified.
 *
 * We are not interested in this interrupt; we just log a debug message.
 */
/*ARGSUSED*/
static int
si_intr_pwr_change(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        SIDBG_P(SIDBG_INTR, si_portp, "si_intr_pwr_change", NULL);
        return (SI_SUCCESS);
}

/*
 * Interrupt which indicates that the PHY state has changed either from
 * Not-Ready to Ready or from Ready to Not-Ready.
 */
static int
si_intr_phy_ready_change(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        sata_device_t sdevice;
        uint32_t SStatus = 0; /* No dev present & PHY not established. */
        int dev_exists_now = 0;
        int dev_existed_previously = 0;

        SIDBG_P(SIDBG_INTR, si_portp,
            "si_intr_phy_rdy_change", NULL);

        mutex_enter(&si_ctlp->sictl_mutex);
        if ((si_ctlp->sictl_sata_hba_tran == NULL) || (si_portp == NULL)) {
                /* the whole controller setup is not yet done. */
                mutex_exit(&si_ctlp->sictl_mutex);
                return (SI_SUCCESS);
        }

        mutex_exit(&si_ctlp->sictl_mutex);

        mutex_enter(&si_portp->siport_mutex);

        /* SStatus tells the presence of device. */
        SStatus = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)PORT_SSTATUS(si_ctlp, port));
        dev_exists_now =
            (SSTATUS_GET_DET(SStatus) == SSTATUS_DET_DEVPRESENT_PHYONLINE);

        if (si_portp->siport_port_type != PORT_TYPE_NODEV) {
                dev_existed_previously = 1;
        }

        bzero((void *)&sdevice, sizeof (sata_device_t));

        sdevice.satadev_addr.cport = (uint8_t)port;
        sdevice.satadev_addr.pmport = PORTMULT_CONTROL_PORT;

        /* we don't have a way of determining the exact port-mult port. */
        if (si_portp->siport_port_type == PORT_TYPE_MULTIPLIER) {
                sdevice.satadev_addr.qual = SATA_ADDR_PMPORT;
        } else {
                sdevice.satadev_addr.qual = SATA_ADDR_CPORT;
        }

        sdevice.satadev_state = SATA_STATE_READY; /* port state */

        if (dev_exists_now) {
                if (dev_existed_previously) {

                        /* Things are fine now. The loss was temporary. */
                        SIDBG_P(SIDBG_INTR, si_portp,
                            "phyrdy: doing BOTH EVENTS TOGETHER", NULL);
                        if (si_portp->siport_active) {
                                SIDBG_P(SIDBG_EVENT, si_portp,
                                    "sending event: LINK_LOST & "
                                    "LINK_ESTABLISHED", NULL);

                                sata_hba_event_notify(
                                    si_ctlp->sictl_sata_hba_tran->\
                                    sata_tran_hba_dip,
                                    &sdevice,
                                    SATA_EVNT_LINK_LOST|
                                    SATA_EVNT_LINK_ESTABLISHED);
                        }

                } else {

                        /* A new device has been detected. */
                        mutex_exit(&si_portp->siport_mutex);
                        si_find_dev_signature(si_ctlp, si_portp, port,
                            PORTMULT_CONTROL_PORT);
                        mutex_enter(&si_portp->siport_mutex);
                        SIDBG_P(SIDBG_INTR, si_portp,
                            "phyrdy: doing ATTACH event", NULL);
                        if (si_portp->siport_active) {
                                SIDBG_P(SIDBG_EVENT, si_portp,
                                    "sending event up: LINK_ESTABLISHED", NULL);

                                sata_hba_event_notify(
                                    si_ctlp->sictl_sata_hba_tran->\
                                    sata_tran_hba_dip,
                                    &sdevice,
                                    SATA_EVNT_LINK_ESTABLISHED);
                        }

                }
        } else { /* No device exists now */

                if (dev_existed_previously) {

                        /* An existing device is lost. */
                        if (si_portp->siport_active) {
                                SIDBG_P(SIDBG_EVENT, si_portp,
                                    "sending event up: LINK_LOST", NULL);

                                sata_hba_event_notify(
                                    si_ctlp->sictl_sata_hba_tran->
                                    sata_tran_hba_dip,
                                    &sdevice,
                                    SATA_EVNT_LINK_LOST);
                        }
                        si_portp->siport_port_type = PORT_TYPE_NODEV;

                } else {

                        /* spurious interrupt */
                        SIDBG_P(SIDBG_INTR, si_portp,
                            "spurious phy ready interrupt", NULL);
                }
        }

        mutex_exit(&si_portp->siport_mutex);
        return (SI_SUCCESS);
}


/*
 * Interrupt which indicates that a COMWAKE OOB signal has been decoded
 * on the receiver.
 *
 * We are not interested in this interrupt; we just log a debug message.
 */
/*ARGSUSED*/
static int
si_intr_comwake_rcvd(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        SIDBG_P(SIDBG_INTR, si_portp,
            "si_intr_commwake_rcvd", NULL);
        return (SI_SUCCESS);
}

/*
 * Interrupt which indicates that the F-bit has been set in SError
 * Diag field.
 *
 * We are not interested in this interrupt; we just log a debug message.
 */
/*ARGSUSED*/
static int
si_intr_unrecognised_fis(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        SIDBG_P(SIDBG_INTR, si_portp,
            "si_intr_unrecognised_fis", NULL);
        return (SI_SUCCESS);
}

/*
 * Interrupt which indicates that the X-bit has been set in SError
 * Diag field.
 *
 * We are not interested in this interrupt; we just log a debug message.
 */
/*ARGSUSED*/
static int
si_intr_dev_xchanged(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{

        SIDBG_P(SIDBG_INTR, si_portp,
            "si_intr_dev_xchanged", NULL);
        return (SI_SUCCESS);
}

/*
 * Interrupt which indicates that the 8b/10b Decode Error counter has
 * exceeded the programmed non-zero threshold value.
 *
 * We are not interested in this interrupt; we just log a debug message.
 */
/*ARGSUSED*/
static int
si_intr_decode_err_threshold(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        SIDBG_P(SIDBG_INTR, si_portp,
            "si_intr_err_threshold", NULL);
        return (SI_SUCCESS);
}

/*
 * Interrupt which indicates that the CRC Error counter has exceeded the
 * programmed non-zero threshold value.
 *
 * We are not interested in this interrupt; we just log a debug message.
 */
/*ARGSUSED*/
static int
si_intr_crc_err_threshold(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        SIDBG_P(SIDBG_INTR, si_portp,
            "si_intr_crc_threshold", NULL);
        return (SI_SUCCESS);
}

/*
 * Interrupt which indicates that the Handshake Error counter has
 * exceeded the programmed non-zero threshold value.
 *
 * We are not interested in this interrupt; we just log a debug message.
 */
/*ARGSUSED*/
static int
si_intr_handshake_err_threshold(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        SIDBG_P(SIDBG_INTR, si_portp,
            "si_intr_handshake_err_threshold", NULL);
        return (SI_SUCCESS);
}

/*
 * Interrupt which indicates that a "Set Device Bits" FIS has been
 * received with N-bit set in the control field.
 *
 * We are not interested in this interrupt; we just log a debug message.
 */
/*ARGSUSED*/
static int
si_intr_set_devbits_notify(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        SIDBG_P(SIDBG_INTR, si_portp,
            "si_intr_set_devbits_notify", NULL);
        return (SI_SUCCESS);
}


/*
 * Enable the interrupts for a particular port.
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static void
si_enable_port_interrupts(si_ctl_state_t *si_ctlp, int port)
{
        uint32_t mask;
        si_port_state_t *si_portp = si_ctlp->sictl_ports[port];

        /* get the current settings first. */
        mask = ddi_get32(si_ctlp->sictl_global_acc_handle,
            (uint32_t *)GLOBAL_CONTROL_REG(si_ctlp));

        SIDBG_P(SIDBG_INIT, si_portp,
            "si_enable_port_interrupts: current mask: 0x%x",
            mask);

        /* enable the bit for current port. */
        SET_BIT(mask, port);

        /* now use this mask to enable the interrupt. */
        ddi_put32(si_ctlp->sictl_global_acc_handle,
            (uint32_t *)GLOBAL_CONTROL_REG(si_ctlp),
            mask);
}

/*
 * Enable interrupts for all the ports.
 */
static void
si_enable_all_interrupts(si_ctl_state_t *si_ctlp)
{
        int port;

        for (port = 0; port < si_ctlp->sictl_num_ports; port++) {
                si_enable_port_interrupts(si_ctlp, port);
        }
}

/*
 * Disable interrupts for a particular port.
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static void
si_disable_port_interrupts(si_ctl_state_t *si_ctlp, int port)
{
        uint32_t mask;

        /* get the current settings first. */
        mask = ddi_get32(si_ctlp->sictl_global_acc_handle,
            (uint32_t *)GLOBAL_CONTROL_REG(si_ctlp));

        /* clear the bit for current port. */
        CLEAR_BIT(mask, port);

        /* now use this mask to disable the interrupt. */
        ddi_put32(si_ctlp->sictl_global_acc_handle,
            (uint32_t *)GLOBAL_CONTROL_REG(si_ctlp),
            mask);

}

/*
 * Disable interrupts for all the ports.
 */
static void
si_disable_all_interrupts(si_ctl_state_t *si_ctlp)
{
        int port;

        for (port = 0; port < si_ctlp->sictl_num_ports; port++) {
                si_disable_port_interrupts(si_ctlp, port);
        }
}

/*
 * Fetches the latest sstatus, scontrol, serror, sactive registers
 * and stuffs them into sata_device_t structure.
 */
static void
fill_dev_sregisters(si_ctl_state_t *si_ctlp, int port, sata_device_t *satadev)
{
        satadev->satadev_scr.sstatus = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_SSTATUS(si_ctlp, port)));
        satadev->satadev_scr.serror = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_SERROR(si_ctlp, port)));
        satadev->satadev_scr.sactive = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_SACTIVE(si_ctlp, port)));
        satadev->satadev_scr.scontrol =
            ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_SCONTROL(si_ctlp, port)));

}

/*
 * si_add_legacy_intrs() handles INTx and legacy interrupts.
 */
static int
si_add_legacy_intrs(si_ctl_state_t *si_ctlp)
{
        dev_info_t      *devinfo = si_ctlp->sictl_devinfop;
        int             actual, count = 0;
        int             x, y, rc, inum = 0;

        SIDBG_C(SIDBG_INIT, si_ctlp, "si_add_legacy_intrs", NULL);

        /* get number of interrupts. */
        rc = ddi_intr_get_nintrs(devinfo, DDI_INTR_TYPE_FIXED, &count);
        if ((rc != DDI_SUCCESS) || (count == 0)) {
                SIDBG_C(SIDBG_ERRS, si_ctlp,
                    "ddi_intr_get_nintrs() failed, "
                    "rc %d count %d\n", rc, count);
                return (DDI_FAILURE);
        }

        /* Allocate an array of interrupt handles. */
        si_ctlp->sictl_intr_size = count * sizeof (ddi_intr_handle_t);
        si_ctlp->sictl_htable = kmem_zalloc(si_ctlp->sictl_intr_size, KM_SLEEP);

        /* call ddi_intr_alloc(). */
        rc = ddi_intr_alloc(devinfo, si_ctlp->sictl_htable, DDI_INTR_TYPE_FIXED,
            inum, count, &actual, DDI_INTR_ALLOC_STRICT);

        if ((rc != DDI_SUCCESS) || (actual == 0)) {
                SIDBG_C(SIDBG_ERRS, si_ctlp,
                    "ddi_intr_alloc() failed, rc %d\n", rc);
                kmem_free(si_ctlp->sictl_htable, si_ctlp->sictl_intr_size);
                return (DDI_FAILURE);
        }

        if (actual < count) {
                SIDBG_C(SIDBG_ERRS, si_ctlp,
                    "Requested: %d, Received: %d", count, actual);

                for (x = 0; x < actual; x++) {
                        (void) ddi_intr_free(si_ctlp->sictl_htable[x]);
                }

                kmem_free(si_ctlp->sictl_htable, si_ctlp->sictl_intr_size);
                return (DDI_FAILURE);
        }

        si_ctlp->sictl_intr_cnt = actual;

        /* Get intr priority. */
        if (ddi_intr_get_pri(si_ctlp->sictl_htable[0],
            &si_ctlp->sictl_intr_pri) != DDI_SUCCESS) {
                SIDBG_C(SIDBG_ERRS, si_ctlp,
                    "ddi_intr_get_pri() failed", NULL);

                for (x = 0; x < actual; x++) {
                        (void) ddi_intr_free(si_ctlp->sictl_htable[x]);
                }

                kmem_free(si_ctlp->sictl_htable, si_ctlp->sictl_intr_size);
                return (DDI_FAILURE);
        }

        /* Test for high level mutex. */
        if (si_ctlp->sictl_intr_pri >= ddi_intr_get_hilevel_pri()) {
                SIDBG_C(SIDBG_ERRS, si_ctlp,
                    "si_add_legacy_intrs: Hi level intr not supported", NULL);

                for (x = 0; x < actual; x++) {
                        (void) ddi_intr_free(si_ctlp->sictl_htable[x]);
                }

                kmem_free(si_ctlp->sictl_htable, sizeof (ddi_intr_handle_t));

                return (DDI_FAILURE);
        }

        /* Call ddi_intr_add_handler(). */
        for (x = 0; x < actual; x++) {
                if (ddi_intr_add_handler(si_ctlp->sictl_htable[x], si_intr,
                    (caddr_t)si_ctlp, NULL) != DDI_SUCCESS) {
                        SIDBG_C(SIDBG_ERRS, si_ctlp,
                            "ddi_intr_add_handler() failed", NULL);

                        for (y = 0; y < actual; y++) {
                                (void) ddi_intr_free(si_ctlp->sictl_htable[y]);
                        }

                        kmem_free(si_ctlp->sictl_htable,
                            si_ctlp->sictl_intr_size);
                        return (DDI_FAILURE);
                }
        }

        /* Call ddi_intr_enable() for legacy interrupts. */
        for (x = 0; x < si_ctlp->sictl_intr_cnt; x++) {
                (void) ddi_intr_enable(si_ctlp->sictl_htable[x]);
        }

        return (DDI_SUCCESS);
}

/*
 * si_add_msictl_intrs() handles MSI interrupts.
 */
static int
si_add_msi_intrs(si_ctl_state_t *si_ctlp)
{
        dev_info_t      *devinfo = si_ctlp->sictl_devinfop;
        int             count, avail, actual;
        int             x, y, rc, inum = 0;

        SIDBG_C(SIDBG_INIT, si_ctlp, "si_add_msi_intrs", NULL);

        /* get number of interrupts. */
        rc = ddi_intr_get_nintrs(devinfo, DDI_INTR_TYPE_MSI, &count);
        if ((rc != DDI_SUCCESS) || (count == 0)) {
                SIDBG_C(SIDBG_ERRS, si_ctlp,
                    "ddi_intr_get_nintrs() failed, "
                    "rc %d count %d\n", rc, count);
                return (DDI_FAILURE);
        }

        /* get number of available interrupts. */
        rc = ddi_intr_get_navail(devinfo, DDI_INTR_TYPE_MSI, &avail);
        if ((rc != DDI_SUCCESS) || (avail == 0)) {
                SIDBG_C(SIDBG_ERRS, si_ctlp,
                    "ddi_intr_get_navail() failed, "
                    "rc %d avail %d\n", rc, avail);
                return (DDI_FAILURE);
        }

        if (avail < count) {
                SIDBG_C(SIDBG_INIT, si_ctlp,
                    "ddi_intr_get_nvail returned %d, navail() returned %d",
                    count, avail);
        }

        /* Allocate an array of interrupt handles. */
        si_ctlp->sictl_intr_size = count * sizeof (ddi_intr_handle_t);
        si_ctlp->sictl_htable = kmem_alloc(si_ctlp->sictl_intr_size, KM_SLEEP);

        /* call ddi_intr_alloc(). */
        rc = ddi_intr_alloc(devinfo, si_ctlp->sictl_htable, DDI_INTR_TYPE_MSI,
            inum, count, &actual, DDI_INTR_ALLOC_NORMAL);

        if ((rc != DDI_SUCCESS) || (actual == 0)) {
                SIDBG_C(SIDBG_ERRS, si_ctlp,
                    "ddi_intr_alloc() failed, rc %d\n", rc);
                kmem_free(si_ctlp->sictl_htable, si_ctlp->sictl_intr_size);
                return (DDI_FAILURE);
        }

        /* use interrupt count returned */
        if (actual < count) {
                SIDBG_C(SIDBG_INIT, si_ctlp,
                    "Requested: %d, Received: %d", count, actual);
        }

        si_ctlp->sictl_intr_cnt = actual;

        /*
         * Get priority for first msi, assume remaining are all the same.
         */
        if (ddi_intr_get_pri(si_ctlp->sictl_htable[0],
            &si_ctlp->sictl_intr_pri) != DDI_SUCCESS) {
                SIDBG_C(SIDBG_ERRS, si_ctlp, "ddi_intr_get_pri() failed", NULL);

                /* Free already allocated intr. */
                for (y = 0; y < actual; y++) {
                        (void) ddi_intr_free(si_ctlp->sictl_htable[y]);
                }

                kmem_free(si_ctlp->sictl_htable, si_ctlp->sictl_intr_size);
                return (DDI_FAILURE);
        }

        /* Test for high level mutex. */
        if (si_ctlp->sictl_intr_pri >= ddi_intr_get_hilevel_pri()) {
                SIDBG_C(SIDBG_ERRS, si_ctlp,
                    "si_add_msi_intrs: Hi level intr not supported", NULL);

                /* Free already allocated intr. */
                for (y = 0; y < actual; y++) {
                        (void) ddi_intr_free(si_ctlp->sictl_htable[y]);
                }

                kmem_free(si_ctlp->sictl_htable, sizeof (ddi_intr_handle_t));

                return (DDI_FAILURE);
        }

        /* Call ddi_intr_add_handler(). */
        for (x = 0; x < actual; x++) {
                if (ddi_intr_add_handler(si_ctlp->sictl_htable[x], si_intr,
                    (caddr_t)si_ctlp, NULL) != DDI_SUCCESS) {
                        SIDBG_C(SIDBG_ERRS, si_ctlp,
                            "ddi_intr_add_handler() failed", NULL);

                        /* Free already allocated intr. */
                        for (y = 0; y < actual; y++) {
                                (void) ddi_intr_free(si_ctlp->sictl_htable[y]);
                        }

                        kmem_free(si_ctlp->sictl_htable,
                            si_ctlp->sictl_intr_size);
                        return (DDI_FAILURE);
                }
        }

        (void) ddi_intr_get_cap(si_ctlp->sictl_htable[0],
            &si_ctlp->sictl_intr_cap);

        if (si_ctlp->sictl_intr_cap & DDI_INTR_FLAG_BLOCK) {
                /* Call ddi_intr_block_enable() for MSI. */
                (void) ddi_intr_block_enable(si_ctlp->sictl_htable,
                    si_ctlp->sictl_intr_cnt);
        } else {
                /* Call ddi_intr_enable() for MSI non block enable. */
                for (x = 0; x < si_ctlp->sictl_intr_cnt; x++) {
                        (void) ddi_intr_enable(si_ctlp->sictl_htable[x]);
                }
        }

        return (DDI_SUCCESS);
}

/*
 * Removes the registered interrupts irrespective of whether they
 * were legacy or MSI.
 */
static void
si_rem_intrs(si_ctl_state_t *si_ctlp)
{
        int x;

        SIDBG_C(SIDBG_INIT, si_ctlp, "si_rem_intrs entered", NULL);

        /* Disable all interrupts. */
        if ((si_ctlp->sictl_intr_type == DDI_INTR_TYPE_MSI) &&
            (si_ctlp->sictl_intr_cap & DDI_INTR_FLAG_BLOCK)) {
                /* Call ddi_intr_block_disable(). */
                (void) ddi_intr_block_disable(si_ctlp->sictl_htable,
                    si_ctlp->sictl_intr_cnt);
        } else {
                for (x = 0; x < si_ctlp->sictl_intr_cnt; x++) {
                        (void) ddi_intr_disable(si_ctlp->sictl_htable[x]);
                }
        }

        /* Call ddi_intr_remove_handler(). */
        for (x = 0; x < si_ctlp->sictl_intr_cnt; x++) {
                (void) ddi_intr_remove_handler(si_ctlp->sictl_htable[x]);
                (void) ddi_intr_free(si_ctlp->sictl_htable[x]);
        }

        kmem_free(si_ctlp->sictl_htable, si_ctlp->sictl_intr_size);
}

/*
 * Resets either the port or the device connected to the port based on
 * the flag variable.
 *
 * The reset effectively throws away all the pending commands. So, the caller
 * has to make provision to handle the pending commands.
 *
 * After the reset, we wait till the port is ready again.
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 *
 * Note: Not port-mult aware.
 */
static int
si_reset_dport_wait_till_ready(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port,
        int flag)
{
        uint32_t port_status;
        int loop_count = 0;
        sata_device_t sdevice;
        uint32_t SStatus;
        uint32_t SControl;
        uint32_t port_intr_status;

        _NOTE(ASSUMING_PROTECTED(si_portp))

        if (flag == SI_PORT_RESET) {
                ddi_put32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)PORT_CONTROL_SET(si_ctlp, port),
                    PORT_CONTROL_SET_BITS_PORT_RESET);

                /* Port reset is not self clearing. So clear it now. */
                ddi_put32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)PORT_CONTROL_CLEAR(si_ctlp, port),
                    PORT_CONTROL_CLEAR_BITS_PORT_RESET);
        } else {
                /* Reset the device. */
                ddi_put32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)PORT_CONTROL_SET(si_ctlp, port),
                    PORT_CONTROL_SET_BITS_DEV_RESET);

                /*
                 * tidbit: this bit is self clearing; so there is no need
                 * for manual clear as we did for port reset.
                 */
        }

        /* Set the reset in progress flag */
        if (!(flag & SI_RESET_NO_EVENTS_UP)) {
                si_portp->siport_reset_in_progress = 1;
        }


        /*
         * Every reset needs a PHY initialization.
         *
         * The way to initialize the PHY is to write a 1 and then
         * a 0 to DET field of SControl register.
         */

        /* Fetch the current SControl before writing the DET part with 1. */
        SControl = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)PORT_SCONTROL(si_ctlp, port));
        SCONTROL_SET_DET(SControl, SCONTROL_DET_COMRESET);
        ddi_put32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_SCONTROL(si_ctlp, port)),
            SControl);
#ifndef __lock_lint
        delay(SI_10MS_TICKS); /* give time for COMRESET to percolate */
#endif /* __lock_lint */

        /* Now fetch the SControl again and rewrite the DET part with 0 */
        SControl = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)PORT_SCONTROL(si_ctlp, port));
        SCONTROL_SET_DET(SControl, SCONTROL_DET_NOACTION);
        ddi_put32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_SCONTROL(si_ctlp, port)),
            SControl);

        /*
         * PHY may be initialized by now. Check the DET field of SStatus
         * to determine if there is a device present.
         *
         * The DET field is valid only if IPM field indicates that
         * the interface is in active state.
         */

        loop_count = 0;
        do {
                SStatus = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)PORT_SSTATUS(si_ctlp, port));

                if (SSTATUS_GET_IPM(SStatus) !=
                    SSTATUS_IPM_INTERFACE_ACTIVE) {
                        /*
                         * If the interface is not active, the DET field
                         * is considered not accurate. So we want to
                         * continue looping.
                         */
                        SSTATUS_SET_DET(SStatus, SSTATUS_DET_NODEV_NOPHY);
                }

                if (loop_count++ > SI_POLLRATE_SSTATUS) {
                        /* We are effectively timing out after 0.1 sec. */
                        break;
                }

                /* Wait for 10 millisec */
#ifndef __lock_lint
                delay(SI_10MS_TICKS);
#endif /* __lock_lint */

        } while (SSTATUS_GET_DET(SStatus) != SSTATUS_DET_DEVPRESENT_PHYONLINE);

        SIDBG_P(SIDBG_POLL_LOOP, si_portp,
            "si_reset_dport_wait_till_ready: loop count: %d, \
                SStatus: 0x%x",
            loop_count,
            SStatus);

        /* Now check for port readiness. */
        loop_count = 0;
        do {
                port_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)PORT_STATUS(si_ctlp, port));

                if (loop_count++ > SI_POLLRATE_PORTREADY) {
                        /* We are effectively timing out after 0.5 sec. */
                        break;
                }

                /* Wait for 10 millisec */
#ifndef __lock_lint
                delay(SI_10MS_TICKS);
#endif /* __lock_lint */

        } while (!(port_status & PORT_STATUS_BITS_PORT_READY));

        SIDBG_P(SIDBG_POLL_LOOP, si_portp,
            "si_reset_dport_wait_till_ready: loop count: %d, \
                port_status: 0x%x, SStatus: 0x%x",
            loop_count,
            port_status,
            SStatus);

        /* Indicate to the framework that a reset has happened. */
        if (!(flag & SI_RESET_NO_EVENTS_UP)) {

                bzero((void *)&sdevice, sizeof (sata_device_t));

                sdevice.satadev_addr.cport = (uint8_t)port;
                sdevice.satadev_addr.pmport = PORTMULT_CONTROL_PORT;

                if (si_portp->siport_port_type == PORT_TYPE_MULTIPLIER) {
                        sdevice.satadev_addr.qual = SATA_ADDR_DPMPORT;
                } else {
                        sdevice.satadev_addr.qual = SATA_ADDR_DCPORT;
                }
                sdevice.satadev_state = SATA_DSTATE_RESET |
                    SATA_DSTATE_PWR_ACTIVE;
                if (si_ctlp->sictl_sata_hba_tran) {
                        sata_hba_event_notify(
                            si_ctlp->sictl_sata_hba_tran->sata_tran_hba_dip,
                            &sdevice,
                            SATA_EVNT_DEVICE_RESET);
                }

                SIDBG_P(SIDBG_EVENT, si_portp,
                    "sending event up: SATA_EVNT_RESET", NULL);
        }

        if ((SSTATUS_GET_IPM(SStatus) == SSTATUS_IPM_INTERFACE_ACTIVE) &&
            (SSTATUS_GET_DET(SStatus) ==
            SSTATUS_DET_DEVPRESENT_PHYONLINE)) {
                /* The interface is active and the device is present */
                if (!(port_status & PORT_STATUS_BITS_PORT_READY)) {
                        /* But the port is is not ready for some reason */
                        SIDBG_P(SIDBG_POLL_LOOP, si_portp,
                            "si_reset_dport_wait_till_ready failed", NULL);
                        return (SI_FAILURE);
                }
        }


        /*
         * For some reason, we are losing the interrupt enablement after
         * any reset condition. So restore them back now.
         */

        SIDBG_P(SIDBG_INIT, si_portp,
            "current interrupt enable set: 0x%x",
            ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)PORT_INTERRUPT_ENABLE_SET(si_ctlp, port)));

        ddi_put32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)PORT_INTERRUPT_ENABLE_SET(si_ctlp, port),
            (INTR_COMMAND_COMPLETE |
            INTR_COMMAND_ERROR |
            INTR_PORT_READY |
            INTR_POWER_CHANGE |
            INTR_PHYRDY_CHANGE |
            INTR_COMWAKE_RECEIVED |
            INTR_UNRECOG_FIS |
            INTR_DEV_XCHANGED |
            INTR_SETDEVBITS_NOTIFY));

        si_enable_port_interrupts(si_ctlp, port);

        /*
         * make sure interrupts are cleared
         */
        port_intr_status = ddi_get32(si_ctlp->sictl_global_acc_handle,
            (uint32_t *)PORT_INTERRUPT_STATUS(si_ctlp, port));

        ddi_put32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_INTERRUPT_STATUS(si_ctlp,
            port)),
            port_intr_status & INTR_MASK);


        SIDBG_P(SIDBG_POLL_LOOP, si_portp,
            "si_reset_dport_wait_till_ready returning success", NULL);

        return (SI_SUCCESS);
}

/*
 * Schedule an initialization of the port using a timeout to get it done
 * off an interrupt thread.
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static void
si_schedule_port_initialize(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port)
{
        si_event_arg_t *args;

        ASSERT(mutex_owned(&si_portp->siport_mutex));

        args = si_portp->siport_event_args;
        if (args->siea_ctlp != NULL) {
                cmn_err(CE_WARN, "si_schedule_port_initialize: "
                    "args->si_ctlp != NULL");
                return;
        }

        args->siea_ctlp = si_ctlp;
        args->siea_port = port;

        (void) timeout(si_do_initialize_port, si_portp, 1);
}

/*
 * Called from timeout()
 * Unpack the arguments and call si_initialize_port_wait_till_ready()
 */
static void
si_do_initialize_port(void *arg)
{
        si_event_arg_t *args;
        si_ctl_state_t *si_ctlp;
        si_port_state_t *si_portp;
        int port;

        si_portp = arg;
        mutex_enter(&si_portp->siport_mutex);

        args = si_portp->siport_event_args;
        si_ctlp = args->siea_ctlp;
        port = args->siea_port;
        args->siea_ctlp = NULL; /* mark siport_event_args as free */
        (void) si_initialize_port_wait_till_ready(si_ctlp, port);

        mutex_exit(&si_portp->siport_mutex);
}


/*
 * Initializes the port.
 *
 * Initialization effectively throws away all the pending commands on
 * the port. So, the caller  has to make provision to handle the pending
 * commands.
 *
 * After the port initialization, we wait till the port is ready again.
 *
 * WARNING, WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static int
si_initialize_port_wait_till_ready(si_ctl_state_t *si_ctlp, int port)
{
        uint32_t port_status;
        int loop_count = 0;
        uint32_t SStatus;
        si_port_state_t *si_portp = si_ctlp->sictl_ports[port];

        /* Initialize the port. */
        ddi_put32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)PORT_CONTROL_SET(si_ctlp, port),
            PORT_CONTROL_SET_BITS_PORT_INITIALIZE);

        /* Wait until Port Ready */
        loop_count = 0;
        do {
                port_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)PORT_STATUS(si_ctlp, port));

                if (loop_count++ > SI_POLLRATE_PORTREADY) {
                        SIDBG_P(SIDBG_INTR, si_portp,
                            "si_initialize_port_wait is timing out: "
                            "port_status: %x",
                            port_status);
                        /* We are effectively timing out after 0.5 sec. */
                        break;
                }

                /* Wait for 10 millisec */
#ifndef __lock_lint
                delay(SI_10MS_TICKS);
#endif /* __lock_lint */

        } while (!(port_status & PORT_STATUS_BITS_PORT_READY));

        SIDBG_P(SIDBG_POLL_LOOP, si_portp,
            "si_initialize_port_wait_till_ready: loop count: %d",
            loop_count);

        SStatus = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)PORT_SSTATUS(si_ctlp, port));

        if ((SSTATUS_GET_IPM(SStatus) == SSTATUS_IPM_INTERFACE_ACTIVE) &&
            (SSTATUS_GET_DET(SStatus) ==
            SSTATUS_DET_DEVPRESENT_PHYONLINE)) {
                /* The interface is active and the device is present */
                if (!(port_status & PORT_STATUS_BITS_PORT_READY)) {
                        /* But the port is is not ready for some reason */
                        return (SI_FAILURE);
                }
        }

        return (SI_SUCCESS);
}


/*
 * si_watchdog_handler() calls us if it detects that there are some
 * commands which timed out. We recalculate the timed out commands once
 * again since some of them may have finished recently.
 */
static void
si_timeout_pkts(
        si_ctl_state_t *si_ctlp,
        si_port_state_t *si_portp,
        int port,
        uint32_t timedout_tags)
{
        uint32_t slot_status;
        uint32_t finished_tags;

        SIDBG_P(SIDBG_TIMEOUT, si_portp,
            "si_timeout_pkts entry", NULL);

        mutex_enter(&si_portp->siport_mutex);
        slot_status = ddi_get32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)(PORT_SLOT_STATUS(si_ctlp, port)));

        si_portp->mopping_in_progress++;

        /*
         * Initialize the controller. The only way to timeout the commands
         * is to reset or initialize the controller. We mop commands after
         * the initialization.
         */
        (void) si_initialize_port_wait_till_ready(si_ctlp, port);

        /*
         * Recompute the timedout tags since some of them may have finished
         * meanwhile.
         */
        finished_tags =  si_portp->siport_pending_tags &
            ~slot_status & SI_SLOT_MASK;
        timedout_tags &= ~finished_tags;

        SIDBG_P(SIDBG_TIMEOUT, si_portp,
            "si_timeout_pkts: finished: %x, timeout: %x",
            finished_tags,
            timedout_tags);

        si_mop_commands(si_ctlp,
            si_portp,
            port,
            slot_status,
            0, /* failed_tags */
            timedout_tags,
            0, /* aborting_tags */
            0);  /* reset_tags */

        mutex_exit(&si_portp->siport_mutex);
}



/*
 * Watchdog handler kicks in every 5 seconds to timeout any commands pending
 * for long time.
 */
static void
si_watchdog_handler(si_ctl_state_t *si_ctlp)
{
        uint32_t pending_tags = 0;
        uint32_t timedout_tags = 0;
        si_port_state_t *si_portp;
        int port;
        int tmpslot;
        sata_pkt_t *satapkt;

        /* max number of cycles this packet should survive */
        int max_life_cycles;

        /* how many cycles this packet survived so far */
        int watched_cycles;

        mutex_enter(&si_ctlp->sictl_mutex);
        SIDBG_C(SIDBG_ENTRY, si_ctlp,
            "si_watchdog_handler entered", NULL);

        for (port = 0; port < si_ctlp->sictl_num_ports; port++) {

                si_portp = si_ctlp->sictl_ports[port];
                if (si_portp == NULL) {
                        continue;
                }

                mutex_enter(&si_portp->siport_mutex);

                if (si_portp->siport_port_type == PORT_TYPE_NODEV) {
                        mutex_exit(&si_portp->siport_mutex);
                        continue;
                }

                /* Skip the check for those ports in error recovery */
                if (si_portp->mopping_in_progress > 0) {
                        SIDBG_P(SIDBG_INFO, si_portp,
                            "si_watchdog_handler: port %d mopping "
                            "in progress, so just return", port);
                        mutex_exit(&si_portp->siport_mutex);
                        continue;
                }

                pending_tags =  si_portp->siport_pending_tags;
                timedout_tags = 0;
                while (pending_tags) {
                        tmpslot = ddi_ffs(pending_tags) - 1;
                        if (tmpslot == -1) {
                                break;
                        }
                        satapkt = si_portp->siport_slot_pkts[tmpslot];

                        if ((satapkt != NULL) && satapkt->satapkt_time) {

                                /*
                                 * We are overloading satapkt_hba_driver_private
                                 * with watched_cycle count.
                                 *
                                 * If a packet has survived for more than it's
                                 * max life cycles, it is a candidate for time
                                 * out.
                                 */
                                watched_cycles = (int)(intptr_t)
                                    satapkt->satapkt_hba_driver_private;
                                watched_cycles++;
                                max_life_cycles = (satapkt->satapkt_time +
                                    si_watchdog_timeout - 1) /
                                    si_watchdog_timeout;
                                if (watched_cycles > max_life_cycles) {
                                        timedout_tags |= (0x1 << tmpslot);
                                        SIDBG_P(SIDBG_TIMEOUT,
                                            si_portp,
                                            "watchdog: timedout_tags: 0x%x",
                                            timedout_tags);
                                }
                                satapkt->satapkt_hba_driver_private =
                                    (void *)(intptr_t)watched_cycles;
                        }

                        CLEAR_BIT(pending_tags, tmpslot);
                }

                if (timedout_tags) {
                        mutex_exit(&si_portp->siport_mutex);
                        mutex_exit(&si_ctlp->sictl_mutex);
                        si_timeout_pkts(si_ctlp, si_portp, port, timedout_tags);
                        mutex_enter(&si_ctlp->sictl_mutex);
                        mutex_enter(&si_portp->siport_mutex);
                }

                mutex_exit(&si_portp->siport_mutex);
        }

        /* Reinstall the watchdog timeout handler. */
        if (!(si_ctlp->sictl_flags & SI_NO_TIMEOUTS)) {
                si_ctlp->sictl_timeout_id =
                    timeout((void (*)(void *))si_watchdog_handler,
                    (caddr_t)si_ctlp, si_watchdog_tick);
        }
        mutex_exit(&si_ctlp->sictl_mutex);
}

/*
 * FMA Functions
 */

/*
 * The IO fault service error handling callback function
 */
/*ARGSUSED*/
static int
si_fm_error_cb(dev_info_t *dip, ddi_fm_error_t *err, const void *impl_data)
{
        /*
         * as the driver can always deal with an error in any dma or
         * access handle, we can just return the fme_status value.
         */
        pci_ereport_post(dip, err, NULL);
        return (err->fme_status);
}

/*
 * si_fm_init - initialize fma capabilities and register with IO
 *              fault services.
 */
static void
si_fm_init(si_ctl_state_t *si_ctlp)
{
        /*
         * Need to change iblock to priority for new MSI intr
         */
        ddi_iblock_cookie_t fm_ibc;

        /* Only register with IO Fault Services if we have some capability */
        if (si_ctlp->fm_capabilities) {
                /* Adjust access and dma attributes for FMA */
                accattr.devacc_attr_access = DDI_FLAGERR_ACC;
                prb_sgt_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;
                buffer_dma_attr.dma_attr_flags |= DDI_DMA_FLAGERR;

                /*
                 * Register capabilities with IO Fault Services.
                 * fm_capabilities will be updated to indicate
                 * capabilities actually supported (not requested.)
                 */
                ddi_fm_init(si_ctlp->sictl_devinfop, &si_ctlp->fm_capabilities,
                    &fm_ibc);

                if (si_ctlp->fm_capabilities == DDI_FM_NOT_CAPABLE)
                        cmn_err(CE_WARN, "si_fm_init: ddi_fm_init fail");

                /*
                 * Initialize pci ereport capabilities if ereport
                 * capable (should always be.)
                 */
                if (DDI_FM_EREPORT_CAP(si_ctlp->fm_capabilities) ||
                    DDI_FM_ERRCB_CAP(si_ctlp->fm_capabilities)) {
                        pci_ereport_setup(si_ctlp->sictl_devinfop);
                }

                /*
                 * Register error callback if error callback capable.
                 */
                if (DDI_FM_ERRCB_CAP(si_ctlp->fm_capabilities)) {
                        ddi_fm_handler_register(si_ctlp->sictl_devinfop,
                            si_fm_error_cb, (void *) si_ctlp);
                }
        }
}

/*
 * si_fm_fini - Releases fma capabilities and un-registers with IO
 *              fault services.
 */
static void
si_fm_fini(si_ctl_state_t *si_ctlp)
{
        /* Only unregister FMA capabilities if registered */
        if (si_ctlp->fm_capabilities) {
                /*
                 * Un-register error callback if error callback capable.
                 */
                if (DDI_FM_ERRCB_CAP(si_ctlp->fm_capabilities)) {
                        ddi_fm_handler_unregister(si_ctlp->sictl_devinfop);
                }

                /*
                 * Release any resources allocated by pci_ereport_setup()
                 */
                if (DDI_FM_EREPORT_CAP(si_ctlp->fm_capabilities) ||
                    DDI_FM_ERRCB_CAP(si_ctlp->fm_capabilities)) {
                        pci_ereport_teardown(si_ctlp->sictl_devinfop);
                }

                /* Unregister from IO Fault Services */
                ddi_fm_fini(si_ctlp->sictl_devinfop);

                /* Adjust access and dma attributes for FMA */
                accattr.devacc_attr_access = DDI_DEFAULT_ACC;
                prb_sgt_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
                buffer_dma_attr.dma_attr_flags &= ~DDI_DMA_FLAGERR;
        }
}

static int
si_check_acc_handle(ddi_acc_handle_t handle)
{
        ddi_fm_error_t de;

        ASSERT(handle != NULL);
        ddi_fm_acc_err_get(handle, &de, DDI_FME_VERSION);
        return (de.fme_status);
}

static int
si_check_dma_handle(ddi_dma_handle_t handle)
{
        ddi_fm_error_t de;

        ASSERT(handle != NULL);
        ddi_fm_dma_err_get(handle, &de, DDI_FME_VERSION);
        return (de.fme_status);
}

static int
si_check_ctl_handles(si_ctl_state_t *si_ctlp)
{
        if ((si_check_acc_handle(si_ctlp->sictl_pci_conf_handle)
            != DDI_SUCCESS) ||
            (si_check_acc_handle(si_ctlp->sictl_global_acc_handle)
            != DDI_SUCCESS) ||
            (si_check_acc_handle(si_ctlp->sictl_port_acc_handle)
            != DDI_SUCCESS)) {
                return (DDI_FAILURE);
        }

        return (DDI_SUCCESS);
}

/*
 * WARNING: The caller is expected to obtain the siport_mutex
 * before calling us.
 */
static int
si_check_port_handles(si_port_state_t *si_portp)
{
        if ((si_check_dma_handle(si_portp->siport_prbpool_dma_handle)
            != DDI_SUCCESS) ||
            (si_check_acc_handle(si_portp->siport_prbpool_acc_handle)
            != DDI_SUCCESS) ||
            (si_check_dma_handle(si_portp->siport_sgbpool_dma_handle)
            != DDI_SUCCESS) ||
            (si_check_acc_handle(si_portp->siport_sgbpool_acc_handle)
            != DDI_SUCCESS)) {
                return (DDI_FAILURE);
        }

        return (DDI_SUCCESS);
}

static void
si_fm_ereport(si_ctl_state_t *si_ctlp, char *detail, char *payload)
{
        uint64_t ena;
        char buf[FM_MAX_CLASS];

        (void) snprintf(buf, FM_MAX_CLASS, "%s.%s", DDI_FM_DEVICE, detail);
        ena = fm_ena_generate(0, FM_ENA_FMT1);

        if (DDI_FM_EREPORT_CAP(si_ctlp->fm_capabilities)) {
                ddi_fm_ereport_post(si_ctlp->sictl_devinfop, buf, ena,
                    DDI_NOSLEEP,
                    FM_VERSION, DATA_TYPE_UINT8, FM_EREPORT_VERSION,
                    "detailed_err_type", DATA_TYPE_STRING, payload,
                    NULL);
        }
}

/*
 * Logs the message.
 */
static void
si_log(si_ctl_state_t *si_ctlp, si_port_state_t *si_portp, char *fmt, ...)
{
        va_list ap;

        mutex_enter(&si_log_mutex);

        va_start(ap, fmt);

        if (si_portp == NULL && si_ctlp == NULL) {
                sata_vtrace_debug(NULL, fmt, ap);
                va_end(ap);
                mutex_exit(&si_log_mutex);
                return;
        }

        if (si_portp == NULL && si_ctlp != NULL) {
                sata_vtrace_debug(si_ctlp->sictl_devinfop, fmt, ap);
                va_end(ap);
                mutex_exit(&si_log_mutex);
                return;
        }

        /*
         * si_portp is not NULL, but si_ctlp might be.
         * Reference si_portp for both port and dip.
         */
        (void) snprintf(si_log_buf, SI_LOGBUF_LEN, "port%d: %s",
            si_portp->siport_port_num, fmt);

        if (si_portp->siport_ctlp == NULL) {
                sata_vtrace_debug(NULL, si_log_buf, ap);
                va_end(ap);
                mutex_exit(&si_log_mutex);
                return;
        }

        sata_vtrace_debug(si_portp->siport_ctlp->sictl_devinfop,
            si_log_buf, ap);

        va_end(ap);

        mutex_exit(&si_log_mutex);

}

static void
si_copy_out_regs(sata_cmd_t *scmd, si_ctl_state_t *si_ctlp, uint8_t port,
    uint8_t slot)
{
        uint32_t *fis_word_ptr;
        si_prb_t *prb;
        int i;
        si_port_state_t *si_portp = si_ctlp->sictl_ports[port];

        /*
         * The LRAM contains the the modified FIS after command completion, so
         * first copy it back to the in-core PRB pool.  To save read cycles,
         * just copy over the FIS portion of the PRB pool.
         */
        prb =  &si_ctlp->sictl_ports[port]->siport_prbpool[slot];

        fis_word_ptr = (uint32_t *)(void *)(&prb->prb_fis);

        for (i = 0; i < (sizeof (fis_reg_h2d_t)/4); i++) {
                fis_word_ptr[i] = ddi_get32(
                    si_ctlp->sictl_port_acc_handle,
                    (uint32_t *)(PORT_LRAM(si_ctlp, port,
                    slot) + i * 4 + 0x08));
        }

        /*
         * always get the status register
         */
        scmd->satacmd_status_reg = GET_FIS_COMMAND(prb->prb_fis);

        DTRACE_PROBE1(satacmd_status_reg, int, scmd->satacmd_status_reg);

        if (scmd->satacmd_flags.sata_copy_out_sec_count_msb) {
                scmd->satacmd_sec_count_msb =
                    GET_FIS_SECTOR_COUNT_EXP(prb->prb_fis);
                SIDBG_P(SIDBG_VERBOSE, si_portp,
                    "copyout satacmd_sec_count_msb %x\n",
                    scmd->satacmd_sec_count_msb);
        }

        if (scmd->satacmd_flags.sata_copy_out_lba_low_msb) {
                scmd->satacmd_lba_low_msb = GET_FIS_SECTOR_EXP(prb->prb_fis);
                SIDBG_P(SIDBG_VERBOSE, si_portp,
                    "copyout satacmd_lba_low_msb %x\n",
                    scmd->satacmd_lba_low_msb);
        }

        if (scmd->satacmd_flags.sata_copy_out_lba_mid_msb) {
                scmd->satacmd_lba_mid_msb = GET_FIS_CYL_LOW_EXP(prb->prb_fis);
                SIDBG_P(SIDBG_VERBOSE, si_portp,
                    "copyout satacmd_lba_mid_msb %x\n",
                    scmd->satacmd_lba_mid_msb);
        }

        if (scmd->satacmd_flags.sata_copy_out_lba_high_msb) {
                scmd->satacmd_lba_high_msb = GET_FIS_CYL_HI_EXP(prb->prb_fis);
                SIDBG_P(SIDBG_VERBOSE, si_portp,
                    "copyout satacmd_lba_high_msb %x\n",
                    scmd->satacmd_lba_high_msb);
        }

        if (scmd->satacmd_flags.sata_copy_out_sec_count_lsb) {
                scmd->satacmd_sec_count_lsb =
                    GET_FIS_SECTOR_COUNT(prb->prb_fis);
                SIDBG_P(SIDBG_VERBOSE, si_portp,
                    "copyout satacmd_sec_count_lsb %x\n",
                    scmd->satacmd_sec_count_lsb);
        }

        if (scmd->satacmd_flags.sata_copy_out_lba_low_lsb) {
                scmd->satacmd_lba_low_lsb = GET_FIS_SECTOR(prb->prb_fis);
                SIDBG_P(SIDBG_VERBOSE, si_portp,
                    "copyout satacmd_lba_low_lsb %x\n",
                    scmd->satacmd_lba_low_lsb);
        }

        if (scmd->satacmd_flags.sata_copy_out_lba_mid_lsb) {
                scmd->satacmd_lba_mid_lsb = GET_FIS_CYL_LOW(prb->prb_fis);
                SIDBG_P(SIDBG_VERBOSE, si_portp,
                    "copyout satacmd_lba_mid_lsb %x\n",
                    scmd->satacmd_lba_mid_lsb);
        }

        if (scmd->satacmd_flags.sata_copy_out_lba_high_lsb) {
                scmd->satacmd_lba_high_lsb = GET_FIS_CYL_HI(prb->prb_fis);
                SIDBG_P(SIDBG_VERBOSE, si_portp,
                    "copyout satacmd_lba_high_lsb %x\n",
                    scmd->satacmd_lba_high_lsb);
        }

        if (scmd->satacmd_flags.sata_copy_out_device_reg) {
                scmd->satacmd_device_reg = GET_FIS_DEV_HEAD(prb->prb_fis);
                SIDBG_P(SIDBG_VERBOSE, si_portp,
                    "copyout satacmd_device_reg %x\n",
                    scmd->satacmd_device_reg);
        }

        if (scmd->satacmd_flags.sata_copy_out_error_reg) {
                scmd->satacmd_error_reg = GET_FIS_FEATURES(prb->prb_fis);
                SIDBG_P(SIDBG_VERBOSE, si_portp,
                    "copyout satacmd_error_reg %x\n",
                    scmd->satacmd_error_reg);
        }
}

/*
 * This function clear the special port by send the PORT RESET
 * After reset was sent, all commands running on the port
 * is aborted
 */
static int
si_clear_port(si_ctl_state_t *si_ctlp, int port)
{

        if (si_ctlp == NULL)
                return (SI_FAILURE);
        /*
         * reset this port so that all existing command
         * is clear
         */
        ddi_put32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)PORT_CONTROL_SET(si_ctlp, port),
            PORT_CONTROL_SET_BITS_PORT_RESET);

        /* Port reset is not self clearing. So clear it now. */
        ddi_put32(si_ctlp->sictl_port_acc_handle,
            (uint32_t *)PORT_CONTROL_CLEAR(si_ctlp, port),
            PORT_CONTROL_CLEAR_BITS_PORT_RESET);
        return (SI_SUCCESS);
}

/*
 * quiesce(9E) entry point.
 * This function is called when the system is single-threaded at high
 * PIL with preemption disabled. Therefore, this function must not be
 * blocked.
 *
 * This function returns DDI_SUCCESS on success, or DDI_FAILURE on failure.
 * DDI_FAILURE indicates an error condition and should almost never happen.
 */
static int
si_quiesce(dev_info_t *dip)
{
        si_ctl_state_t *si_ctlp;
        int instance;
        int port;

        instance = ddi_get_instance(dip);
        si_ctlp = ddi_get_soft_state(si_statep, instance);
        if (si_ctlp == NULL)
                return (DDI_FAILURE);

        SIDBG_C(SIDBG_ENTRY, si_ctlp, "si_quiesce enter", NULL);
        /*
         * Disable all the interrupts before quiesce
         */

        for (port = 0; port < si_ctlp->sictl_num_ports; port++) {
                si_disable_port_interrupts(si_ctlp, port);
                (void) si_clear_port(si_ctlp, port);
        }

        return (DDI_SUCCESS);
}