/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2026 Oxide Computer Company
 */

#include "ena_hw.h"
#include "ena.h"

/*
 * Elastic Network Adapter (ENA) Driver
 * ------------------------------------
 *
 * The ena driver provides support for the AWS ENA device, also
 * referred to as their "enhanced networking". This device is present
 * on "Nitro"-based instances. It presents itself with the following
 * PCI Vendor/Device IDs
 *
 *    o 1d0f:0ec2 -- ENA PF
 *    o 1d0f:1ec2 -- ENA PF (Reserved)
 *    o 1d0f:ec20 -- ENA VF
 *    o 1d0f:ec21 -- ENA VF (Reserved)
 *
 * This driver provides support for only the essential features needed
 * to drive traffic on an ENA device. Support for the following
 * features IS NOT currently implemented.
 *
 *    o Admin Queue Interrupts: queue completion events are always polled
 *    o FMA
 *    o Rx checksum offloads
 *    o Tx checksum offloads
 *    o Tx DMA bind (borrow buffers)
 *    o Rx DMA bind (loaned buffers)
 *    o TSO
 *    o RSS
 *    o Support for different Tx completion policies
 *    o More controlled Tx recycling and Rx refill
 *
 * Even without these features the ena driver should perform
 * reasonably well.
 *
 * Driver vs. Hardware Types
 * -------------------------
 *
 * To properly communicate with the ENA device the driver must
 * populate memory (registers and buffers) with specific types. These
 * types are defined by the device and are found under the "common"
 * (ena_com) code of the AWS Linux and FreeBSD drivers [1]. We have
 * simplified this a bit by defining all device-specific types in the
 * ena_hw.h file. Furthermore, all device-specific types are given an
 * "enahw" prefix. This makes it clear when we are dealing with a
 * device type and when we are dealing with a driver type.
 *
 * [1]: https://github.com/amzn/amzn-drivers
 *
 * Groups, Rings (Queues), and Interrupts
 * --------------------------------------
 *
 * The ENA device presents one mac group. This single mac group
 * represents the single unicast address that this device represents
 * in your AWS instance. The ENA device presents no option for
 * configuring additional MAC addresses, multicast, or promisc mode --
 * you receive only what AWS wants you to receive.
 *
 * This single mac group may have one or more rings. The ENA driver
 * refers to rings as queues, for no special reason other than it was
 * the dominant language in the Linux and FreeBSD drivers, and it
 * spilled over into this port. The upper bound on number of queues is
 * presented by the device. However, we don't just go with whatever
 * number of queues the device reports; but rather we limit the queues
 * based on other factors such as an absolute maximum, number of
 * online CPUs, and number of available interrupts. The upper bound is
 * calculated by ena_set_max_io_queues(), and that is used and
 * possibly further restricted in ena_attach_intr_alloc(). As this
 * point, ultimately, it is the number of available interrupts (minus
 * one for the admin queue) that determines the number of queues: one
 * Tx and one Rx on each I/O interrupt.
 *
 * NOTE: Perhaps it is overly restrictive to limit the number of
 * queues to the number of I/O interrupts. Something worth considering
 * on larger instances if they present far less interrupts than they
 * do queues + CPUs.
 *
 * The ENA device presents MSI-X interrupts only. During attach the
 * driver queries the number of available interrupts and sets aside
 * one for admin/AENQ (vector 0) and the rest for I/O (vector 1 to N).
 * This means that a Tx/Rx queue at index 0 will map to vector 1, and
 * so on.
 *
 * NOTE: The ENA driver currently doesn't make full use of the Admin
 * Queue interrupt. This interrupt is used both to notify the driver
 * when a command response is ready, and when an async event is posted.
 * The ENA driver always polls the Admin Queue for responses.
 *
 * Tx Queue Workings
 * -----------------
 *
 * A single Tx queue (ena_txq_t) is made up of one submission queue
 * (SQ) and its paired completion queue (CQ). These two queues form a
 * logical descriptor ring which is used to send packets out of the
 * device -- where each SQ entry describes the packet to be sent
 * (enahw_tx_desc_t) and each CQ entry describes the result of sending
 * a packet (enahw_tx_cdesc_t). For this to work the host and device
 * must agree on which descriptors are currently owned by the host
 * (free for sending) and which are owned by the device (pending
 * device completion). This state is tracked on the host side via head
 * and tail indexes along with a phase value.
 *
 * The head and tail values represent the head and tail of the FIFO
 * queue of pending packets -- the next packet to be sent by the
 * device is head, and all descriptors up to tail are ready for
 * sending. The phase allows the host to determine which CQ
 * descriptors represent completed events when using per-SQ completion
 * events (as opposed to queue head pointer updates). As the queues
 * represent a logical ring buffer, the phase must alternate on
 * wrap-around. The device initializes the phase to zero, and the host
 * starts with a phase of 1. The first packet descriptor writes, and
 * their corresponding completions, are indicated with a phase of 1.
 *
 *
 * For example, the diagram below represents the SQ/CQ state after the
 * first 6 packets have been sent by the host and 2 of them have been
 * completed by the device (and these completions have been processed
 * by the driver). In this state the host could send 4 more packets
 * before needing to wait on completion events.
 *
 *
 *    +---+---+---+---+---+---+---+---+
 * SQ | 1 | 1 | 1 | 1 | 1 | 1 | 0 | 0 |   phase = 1
 *    +---+---+---+---+---+---+---+---+
 *                              ^
 *                              |
 *                            tail
 *            head
 *              |
 *              v
 *    +---+---+---+---+---+---+---+---+
 * CQ | 1 | 1 | 0 | 0 | 0 | 0 | 0 | 0 |   phase = 1
 *    +---+---+---+---+---+---+---+---+
 *
 *
 * The next diagram shows how the state changes as 5 more packets are
 * sent (for a total of 11) and 7 more are completed (for a total of
 * 9). Notice that as the SQ and CQ have wrapped around their phases
 * have been complemented. In this state the host could send 6 more
 * packets before needing to wait on completion events.
 *
 *    +---+---+---+---+---+---+---+---+
 * SQ | 0 | 0 | 0 | 1 | 1 | 1 | 1 | 1 |   phase = 0
 *    +---+---+---+---+---+---+---+---+
 *                  ^
 *                  |
 *                tail
 *        head
 *          |
 *          v
 *    +---+---+---+---+---+---+---+---+
 * CQ | 0 | 1 | 1 | 1 | 1 | 1 | 1 | 1 |   phase = 0
 *    +---+---+---+---+---+---+---+---+
 *
 *
 * Currently, all packets are copied for Tx. At ring start we allocate
 * a Tx Control Buffer (TCB) for each queue descriptor. Each TCB has
 * DMA buffer associated with it; and each buffer is large enough to
 * hold the MTU. Therefore, Tx descriptors and TCBs currently have a
 * 1:1 mapping. When a packet is sent, the mblk's buffer is copied to
 * the TCB's DMA buffer, and a new descriptor is written to the SQ
 * describing said TCB buffer. If and when we add more advanced
 * features like DMA binding of mblks and TSO, this 1:1 guarantee will
 * no longer hold.
 *
 * Low Latency Queues (LLQ)
 * ------------------------
 *
 * The ENA device supports two placement policies for Tx submission
 * queue descriptors: host memory (PLACEMENT_POLICY_HOST) and device
 * memory (PLACEMENT_POLICY_DEV). With host placement, the driver
 * writes descriptors to host memory and the device reads them via
 * DMA. With device placement, the driver writes descriptors directly
 * to a region of device memory mapped through PCI BAR 2. This is
 * known as LLQ mode and reduces latency by eliminating the DMA read.
 *
 * Some device generations require LLQ. In particular, Nitro v5 and
 * later instances will not attach without it. When the device
 * supports LLQ, the driver negotiates LLQ parameters during attach
 * via the ENA_ADMIN_LLQ feature. If the device does not support LLQ,
 * the driver uses host placement. If the device supports LLQ but
 * negotiation fails, the driver fails to attach.
 *
 * The following LLQ parameters are negotiated with the device
 * during attach:
 *
 *    o Entry size: we honour the device's recommended entry size
 *      if it is supported. If the device has no recommendation we
 *      fall back to the lowest supported size. Each SQ entry is this
 *      size, comprising descriptors followed by inline packet header
 *      data. Larger entries allow more header to be inlined but
 *      reduce the maximum queue depth since the LLQ memory is
 *      fixed-size.
 *
 *    o Headers inline in the first entry: the packet header is
 *      placed in the first SQ entry immediately after the
 *      descriptors, using the remaining space in the entry.
 *
 *    o Descriptors before header: the number of descriptors that
 *      precede the inline header in the first entry is negotiated
 *      with the device, preferring 2, then 1, then 4, then 8. This
 *      is the same order as other ena driver common code, with 2
 *      being preferred over 1 as it maximises inline header space
 *      while still allowing a meta + data descriptor pair.
 *
 *    o Stride control: prefers multiple descriptors per entry,
 *      falling back to single descriptor per entry.
 *
 * The header location is fixed as inline, which is the only mode
 * current ENA hardware supports. The negotiation fails attach if
 * the device does not support any of the driver's choices for a
 * given parameter.
 *
 * Accelerated LLQ
 * ---------------
 *
 * Newer devices (Nitro v5/v6) additionally require accelerated LLQ
 * mode, which the driver enables by setting two flags in the
 * SET_FEATURE command:
 *
 *    o DISABLE_META_CACHING: the device will not cache Tx meta
 *      descriptors, so the driver must include a meta descriptor
 *      with every packet. This simplifies device state tracking
 *      at the cost of additional descriptor space per packet.
 *
 *    o LIMIT_TX_BURST: the device specifies a maximum number of
 *      LLQ entries that may be written between doorbells. The
 *      driver tracks this per-queue and rings the doorbell when
 *      the limit is reached, even if more packets are ready.
 *
 * LLQ Tx Path
 * -----------
 *
 * In LLQ mode, the Tx path differs from the host placement path
 * described above. Each queue has a bounce buffer -- a single
 * staging area the size of one LLQ entry. For each packet the
 * driver composes the complete LLQ entry in the bounce buffer:
 *
 *    +--------+------//------+---------------------------+
 *    | meta   | data descs   | inline packet header      |
 *    | 16B    | N x 16B      | remaining bytes           |
 *    +--------+------//------+---------------------------+
 *    |<------------- entry_size bytes ------------------>|
 *
 * The meta descriptor carries per-packet metadata (L3 header
 * offset, MSS, etc.). The data descriptor(s) are the same as in
 * host mode; N is the negotiated descs_before_header value. The
 * inline header is copied from the packet's mblk chain and allows
 * the device to begin processing before the full DMA payload
 * arrives.
 *
 * Once the bounce buffer is complete, it is written to device
 * memory using 64-bit stores. The bounce buffer approach ensures
 * that the device sees a complete, consistent entry rather than
 * a partial one that could result from writing fields individually.
 *
 * The LLQ BAR is mapped write-combining (DDI_STORECACHING_OK_ACC)
 * so the CPU may merge and reorder stores within an entry. This
 * is safe because the device does not poll its local memory for
 * phase bit transitions the way a traditional NIC scans a
 * host-memory descriptor ring; it only inspects LLQ entries
 * after receiving a doorbell write. A membar_producer() before
 * the doorbell guarantees that all prior stores to the LLQ entry
 * are ordered before the doorbell, so there is no need to write
 * phase bits in reverse order or otherwise arrange for the phase
 * to be the last thing written.
 *
 * There is no DMA involved for the descriptors themselves; they
 * reach the device via MMIO writes through the BAR. The packet
 * payload beyond the inline header is still transferred by DMA
 * and is synced separately (see ena_tcb_pull()).
 *
 * Rx Queue Workings
 * -----------------
 *
 * In terms of implementing the logical descriptor ring, the Rx queues
 * are very much like the Tx queues. There is a paired SQ and CQ for
 * each logical ring. The difference is that in Rx the SQ is for
 * handing buffers to the device to fill, and the CQ is for describing
 * the contents of those buffers for a given received frame. At Rx
 * ring start we allocate a Rx Control Buffer (RCB) for each
 * descriptor in the ring. Each RCB has a DMA buffer associated with
 * it; and each buffer is large enough to hold the MTU. For each
 * received frame we copy the contents out of the RCB and into its own
 * mblk, immediately returning the RCB for reuse. As with Tx, this
 * gives us a simple 1:1 mapping currently, but if more advanced
 * features are implemented later this could change.
 *
 * Asynchronous Event Notification Queue (AENQ)
 * --------------------------------------------
 *
 * Each ENA device comes with a mechanism for sending out-of-band
 * notifications to the driver. This includes events like link state
 * changes, fatal errors, and a watchdog/keep alive signal. The AENQ
 * delivery mechanism is via interrupt, handled by the ena_aenq_work()
 * function, which dispatches via the eaenq_hdlrs table. If no handler
 * is registered, the ena_aenq_default_hdlr() handler is used. A given
 * device may not support all the different event types
 * (enahw_aenq_groups_t); and the driver may choose to enable a subset
 * of the supported events. During attach we call ena_aenq_configure()
 * to negotiate the supported/enabled events. The enabled group is
 * stored at ena_aenq_enabled_groups.
 *
 * Queues and Unsigned Wraparound
 * ------------------------------
 *
 * All the queues use a uint16_t value as their head/tail values, e.g.
 * the Rx queue's er_cq_head_idx value. You might notice that we only
 * ever increment these values, letting them perform implicit unsigned
 * integer wraparound. This is intended. This is the same behavior as
 * the common code, and seems to be what the hardware expects. Of
 * course, when accessing our own descriptor arrays we must make sure
 * to first perform a modulo of this value or risk running off into
 * space.
 *
 * Watchdog and Device Reset
 * -------------------------
 *
 * While the device is running, the driver periodically invokes a
 * watchdog function to check that all is well, and to reset the
 * device if not. The device will be reset if any of the following is
 * true:
 *
 *    o The device's status register fatal error bit is set. A device
 *      in this state will no longer process any queues;
 *    o No asynchronous event keepalives have been received for some
 *      time -- see ENA_DEVICE_KEEPALIVE_TIMEOUT_NS;
 *    o A Tx queue has remained blocked for some time -- see
 *      ENA_TX_STALL_TIMEOUT;
 *    o The device has requested, via an asynchronous event, that we
 *      perform a reset;
 *    o Driver code has detected an error and set the EN_STATE_ERROR
 *      bit in ena_state.
 *
 * There is a "fatal error" asynchronous event, but common code does
 * not use that as a reason to trigger a reset, and so neither do we.
 *
 * The global `ena_force_reset` variable can be used as a simple means
 * to trigger a reset during driver development and testing. If there
 * are multiple instances, it is likely that only one of them will
 * reset when this variable is changed to `true`.
 *
 * Attach Sequencing
 * -----------------
 *
 * Most drivers implement their attach/detach/cleanup functions as a
 * sequential stream of function calls used to allocate and initialize
 * resources in an order determined by the device's programming manual
 * combined with any requirements imposed by the kernel and its
 * relevant modules. These functions can become quite long. It is
 * often hard to see the order in which steps are taken, and even
 * harder to tell if detach/cleanup undoes them in the correct order,
 * or even if it undoes them at all! The only sure way to understand
 * the flow is to take good notes while closely inspecting each line
 * of code. Even then, it's easy for attach and detach to get out of
 * sync.
 *
 * Some more recent drivers have improved on this situation by using a
 * bit vector to track the sequence of events in attach/detach. Each
 * bit is declared in as an enum value, in the same order it is
 * expected attach would run, and thus detach would run in the exact
 * opposite order. This has three main benefits:
 *
 *    1. It makes it easier to determine sequence order at a
 *       glance.
 *
 *    2. It gives a better idea of what state the device is in during
 *       debugging (the sequence bit vector is kept with the instance
 *       state).
 *
 *    3. The detach function can verify that all sequence bits are
 *       cleared, indicating that everything done in attach was
 *       successfully undone.
 *
 * These are great improvements. However, the attach/detach functions
 * can still become unruly, and there is still no guarantee that
 * detach is done in opposite order of attach (this is not always
 * strictly required, but is probably the best way to write detach).
 * There is still a lot of boilerplate and chance for programmer
 * error.
 *
 * The ena driver takes the sequence idea a bit further, creating a
 * descriptor table of the attach sequence (ena_attach_tbl). This
 * table is used by attach/detach to generically, declaratively, and
 * programmatically enforce the precise sequence order and verify that
 * anything that is done is undone. This provides several benefits:
 *
 *    o Correct order is enforced implicitly by the descriptor table.
 *      It is impossible for the detach sequence to run in any other
 *      order other than opposite that of attach.
 *
 *    o It is obvious what the precise attach sequence is. While the
 *      bit vector enum helps a lot with this it doesn't prevent
 *      programmer error. With the sequence defined as a declarative
 *      table it makes it easy for the programmer to see the order and
 *      know it's followed exactly.
 *
 *    o It is impossible to modify the attach sequence without also
 *      specifying a callback for its dual in the detach sequence.
 *
 *    o Common and repetitive code like error checking, logging, and bit
 *      vector modification is eliminated and centralized, again
 *      reducing the chance of programmer error.
 *
 * The ena attach sequence is defined under ena_attach_seq_t. The
 * descriptor table is defined under ena_attach_tbl.
 */

/*
 * These are some basic data layout invariants on which development
 * assumptions where made.
 */
CTASSERT(sizeof (enahw_tx_data_desc_t) == 16);
CTASSERT(sizeof (enahw_tx_data_desc_t) == sizeof (enahw_tx_meta_desc_t));
CTASSERT(sizeof (enahw_tx_data_desc_t) == sizeof (enahw_tx_desc_t));
CTASSERT(sizeof (enahw_tx_meta_desc_t) == sizeof (enahw_tx_desc_t));

/*
 * Amazon does not specify the endianess of the ENA device. We assume
 * it's the same as the bus, and we assume the CPU/bus is always
 * little endian.
 */
#ifdef _BIG_ENDIAN
#error "ENA driver is little-endian only"
#endif

uint64_t ena_admin_cmd_timeout_ns = ENA_ADMIN_CMD_DEF_TIMEOUT_NS;

/*
 * Log an error message. We leave the destination (console or system
 * log) up to the caller
 */
void
ena_err(const ena_t *ena, const char *fmt, ...)
{
        va_list ap;

        va_start(ap, fmt);
        if (ena != NULL && ena->ena_dip != NULL) {
                vdev_err(ena->ena_dip, CE_WARN, fmt, ap);
        } else {
                vcmn_err(CE_WARN, fmt, ap);
        }
        va_end(ap);
}

void
ena_panic(const ena_t *ena, const char *fmt, ...)
{
        va_list ap;

        va_start(ap, fmt);
        if (ena != NULL && ena->ena_dip != NULL) {
                vdev_err(ena->ena_dip, CE_PANIC, fmt, ap);
        } else {
                vcmn_err(CE_PANIC, fmt, ap);
        }
        va_end(ap);
}

/*
 * Set this to true to enable debug messages.
 */
bool ena_debug = false;

/*
 * Log a debug message. We force all debug messages to go to the
 * system log.
 */
void
ena_dbg(const ena_t *ena, const char *fmt, ...)
{
        va_list ap;

        if (ena_debug) {
                char msg[1024];

                va_start(ap, fmt);
                (void) vsnprintf(msg, sizeof (msg), fmt, ap);
                va_end(ap);

                if (ena != NULL && ena->ena_dip != NULL) {
                        dev_err(ena->ena_dip, CE_NOTE, "!%s", msg);
                } else {
                        cmn_err(CE_NOTE, "!%s", msg);
                }
        }
}

void
ena_trigger_reset(ena_t *ena, enahw_reset_reason_t reason)
{
        mutex_enter(&ena->ena_lock);
        ena->ena_reset_reason = reason;
        mutex_exit(&ena->ena_lock);
        atomic_or_32(&ena->ena_state, ENA_STATE_ERROR);
}

/*
 * Determine if a given feature is available on this device.
 */
bool
ena_is_feat_avail(ena_t *ena, const enahw_feature_id_t feat_id)
{
        VERIFY3U(feat_id, <=, ENAHW_FEAT_NUM);
        uint32_t mask = 1U << feat_id;

        /*
         * The device attributes feature is always supported, as
         * indicated by the common code. Host attributes must be sent
         * before querying device attributes, so the supported features
         * bitmask is not yet populated at that point.
         */
        if (feat_id == ENAHW_FEAT_DEVICE_ATTRIBUTES ||
            feat_id == ENAHW_FEAT_HOST_ATTR_CONFIG) {
                return (true);
        }

        return ((ena->ena_supported_features & mask) != 0);
}

/*
 * Determine if a given capability is available on this device.
 */
bool
ena_is_cap_avail(ena_t *ena, const enahw_capability_id_t cap_id)
{
        VERIFY3U(cap_id, <=, ENAHW_CAP_NUM);
        uint32_t mask = 1U << cap_id;

        return ((ena->ena_capabilities & mask) != 0);
}

static bool
ena_device_reset(ena_t *ena, enum enahw_reset_reason_types reason)
{
        uint32_t rval, wval, reason_lsb, reason_msb;
        hrtime_t timeout, expired;

        rval = ena_hw_bar_read32(ena, ENAHW_REG_DEV_STS);
        if ((rval & ENAHW_DEV_STS_READY_MASK) == 0) {
                ena_err(ena, "reset: device is not ready");
                return (false);
        }

        rval = ena_hw_bar_read32(ena, ENAHW_REG_CAPS);

        /*
         * The device stores the reset timeout at 100ms resolution; we
         * normalize that to nanoseconds.
         */
        timeout = MSEC2NSEC(ENAHW_CAPS_RESET_TIMEOUT(rval) * 100);

        if (timeout == 0) {
                ena_err(ena, "device gave invalid (0) reset timeout");
                return (false);
        }

        expired = gethrtime() + timeout;

        wval = ENAHW_DEV_CTL_DEV_RESET_MASK;

        reason_lsb = ENAHW_RESET_REASON_LSB(reason);
        reason_msb = ENAHW_RESET_REASON_MSB(reason);

        wval |= (reason_lsb << ENAHW_DEV_CTL_RESET_REASON_SHIFT) &
            ENAHW_DEV_CTL_RESET_REASON_MASK;
        if (ena_is_cap_avail(ena, ENAHW_CAP_EXTENDED_RESET_REASONS)) {
                wval |= (reason_msb << ENAHW_DEV_CTL_RESET_REASON_EXT_SHIFT) &
                    ENAHW_DEV_CTL_RESET_REASON_EXT_MASK;
        } else if (reason_msb != 0) {
                /* Fall back to "generic" which we know will fit */
                wval = ENAHW_DEV_CTL_DEV_RESET_MASK;
                wval |= (ENAHW_RESET_GENERIC <<
                    ENAHW_DEV_CTL_RESET_REASON_SHIFT) &
                    ENAHW_DEV_CTL_RESET_REASON_MASK;
        }

        ena_hw_bar_write32(ena, ENAHW_REG_DEV_CTL, wval);

        /*
         * Make sure reset is in progress.
         */
        for (;;) {
                rval = ena_hw_bar_read32(ena, ENAHW_REG_DEV_STS);

                if ((rval & ENAHW_DEV_STS_RESET_IN_PROGRESS_MASK) != 0)
                        break;

                if (gethrtime() > expired) {
                        ena_err(ena, "device reset start timed out");
                        return (false);
                }

                /* Sleep for 100 milliseconds. */
                delay(drv_usectohz(100 * 1000));
        }

        /*
         * Reset the timeout counter for the next device request.
         */
        expired = gethrtime() + timeout;

        /*
         * Wait for the device reset to finish.
         */
        ena_hw_bar_write32(ena, ENAHW_REG_DEV_CTL, 0);
        for (;;) {
                rval = ena_hw_bar_read32(ena, ENAHW_REG_DEV_STS);

                if ((rval & ENAHW_DEV_STS_RESET_IN_PROGRESS_MASK) == 0) {
                        break;
                }

                if (gethrtime() > expired) {
                        ena_err(ena, "device reset timed out");
                        return (false);
                }

                /* Sleep for 100 milliseconds. */
                delay(drv_usectohz(100 * 1000));
        }

        ena_dbg(ena, "device reset succeeded");

        return (true);
}

static bool
ena_attach_pci(ena_t *ena)
{
        ddi_acc_handle_t hdl;

        if (pci_config_setup(ena->ena_dip, &hdl) != 0) {
                return (false);
        }

        ena->ena_pci_hdl = hdl;
        ena->ena_pci_vid = pci_config_get16(hdl, PCI_CONF_VENID);
        ena->ena_pci_did = pci_config_get16(hdl, PCI_CONF_DEVID);
        ena->ena_pci_rev = pci_config_get8(hdl, PCI_CONF_REVID);
        ena->ena_pci_svid = pci_config_get16(hdl, PCI_CONF_SUBVENID);
        ena->ena_pci_sdid = pci_config_get16(hdl, PCI_CONF_SUBSYSID);
        ena_dbg(ena, "vid: 0x%x did: 0x%x rev: 0x%x svid: 0x%x sdid: 0x%x",
            ena->ena_pci_vid, ena->ena_pci_did, ena->ena_pci_rev,
            ena->ena_pci_svid, ena->ena_pci_sdid);

        return (true);
}

static void
ena_cleanup_pci(ena_t *ena, bool resetting)
{
        VERIFY0(resetting);
        pci_config_teardown(&ena->ena_pci_hdl);
}

static void
ena_cleanup_regs_map(ena_t *ena, bool resetting)
{
        VERIFY0(resetting);
        ddi_regs_map_free(&ena->ena_reg_hdl);
}

/*
 * Map a PCI BAR number (0-5) to a DDI register set number.
 */
static int
ena_bar_to_rnumber(ena_t *ena, uint8_t bar)
{
        pci_regspec_t *regs;
        uint_t bar_offset, regs_length, rcount;
        int rnumber = -1;

        if (bar > 5)
                return (-1);

        /*
         * PCI_CONF_BASE0 is 0x10; each BAR is 4 bytes apart.
         */
        bar_offset = PCI_CONF_BASE0 + sizeof (uint32_t) * bar;

        if (ddi_prop_lookup_int_array(DDI_DEV_T_ANY, ena->ena_dip,
            DDI_PROP_DONTPASS, "reg", (int **)&regs, &regs_length) !=
            DDI_PROP_SUCCESS) {
                return (-1);
        }

        rcount = regs_length * sizeof (int) / sizeof (pci_regspec_t);

        for (int i = 0; i < rcount; i++) {
                if (PCI_REG_REG_G(regs[i].pci_phys_hi) == bar_offset) {
                        rnumber = i;
                        break;
                }
        }

        ddi_prop_free(regs);

        return (rnumber);
}

static bool
ena_attach_regs_map(ena_t *ena)
{
        ddi_device_acc_attr_t attr;
        int rnumber;
        int ret = 0;

        rnumber = ena_bar_to_rnumber(ena, ENA_REG_BAR);
        if (rnumber == -1) {
                ena_err(ena, "failed to find rnumber for BAR %d", ENA_REG_BAR);
                return (false);
        }

        if (ddi_dev_regsize(ena->ena_dip, rnumber, &ena->ena_reg_size) !=
            DDI_SUCCESS) {
                ena_err(ena, "failed to get register set %d size", rnumber);
                return (false);
        }

        ena_dbg(ena, "register size: %ld", ena->ena_reg_size);
        bzero(&attr, sizeof (attr));
        attr.devacc_attr_version = DDI_DEVICE_ATTR_V1;
        attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
        attr.devacc_attr_dataorder = DDI_STRICTORDER_ACC;
        attr.devacc_attr_access = DDI_DEFAULT_ACC;

        /*
         * This function can return several different failure values,
         * so we make sure to capture its return value for the purpose
         * of logging.
         */
        ret = ddi_regs_map_setup(ena->ena_dip, rnumber,
            &ena->ena_reg_base, 0, ena->ena_reg_size, &attr,
            &ena->ena_reg_hdl);

        if (ret != DDI_SUCCESS) {
                ena_err(ena, "failed to map BAR %d (reg %d): %d",
                    ENA_REG_BAR, rnumber, ret);
                return (false);
        }

        ena_dbg(ena, "registers BAR %d mapped to base: 0x%p size: %ld",
            ENA_REG_BAR, (void *)ena->ena_reg_base, ena->ena_reg_size);

        return (true);
}

/*
 * Free any resources related to the admin submission queue.
 */
static void
ena_admin_sq_free(ena_t *ena)
{
        ena_dma_free(&ena->ena_aq.ea_sq.eas_dma);
}

/*
 * Initialize the admin submission queue.
 */
static bool
ena_admin_sq_init(ena_t *ena)
{
        ena_adminq_t *aq = &ena->ena_aq;
        ena_dma_buf_t *dma = &aq->ea_sq.eas_dma;
        size_t size = aq->ea_qlen * sizeof (*aq->ea_sq.eas_entries);
        uint32_t addr_low, addr_high, wval;

        if (aq->ea_sq.eas_entries == NULL) {
                ena_dma_conf_t conf = {
                        .edc_size = size,
                        .edc_align = ENAHW_ADMIN_SQ_DESC_BUF_ALIGNMENT,
                        .edc_sgl = 1,
                        .edc_endian = DDI_NEVERSWAP_ACC,
                        .edc_stream = false,
                };

                if (!ena_dma_alloc(ena, dma, &conf, size)) {
                        ena_err(ena, "failed to allocate DMA for Admin SQ");
                        return (false);
                }

                ENA_DMA_VERIFY_ADDR(ena, dma->edb_cookie->dmac_laddress);
                aq->ea_sq.eas_entries = (void *)dma->edb_va;
        } else {
                ena_dma_bzero(dma);
        }

        aq->ea_sq.eas_tail = 0;
        aq->ea_sq.eas_phase = 1;
        aq->ea_sq.eas_dbaddr =
            (uint32_t *)(ena->ena_reg_base + ENAHW_REG_ASQ_DB);
        addr_low = (uint32_t)(dma->edb_cookie->dmac_laddress);
        addr_high = (uint32_t)(dma->edb_cookie->dmac_laddress >> 32);
        ena_hw_bar_write32(ena, ENAHW_REG_ASQ_BASE_LO, addr_low);
        ena_hw_bar_write32(ena, ENAHW_REG_ASQ_BASE_HI, addr_high);
        wval = ENAHW_ASQ_CAPS_DEPTH(aq->ea_qlen) |
            ENAHW_ASQ_CAPS_ENTRY_SIZE(sizeof (*aq->ea_sq.eas_entries));
        ena_hw_bar_write32(ena, ENAHW_REG_ASQ_CAPS, wval);

        return (true);
}

/*
 * Free any resources related to the admin completion queue.
 */
static void
ena_admin_cq_free(ena_t *ena)
{
        ena_dma_free(&ena->ena_aq.ea_cq.eac_dma);
}

/*
 * Initialize the admin completion queue.
 */
static bool
ena_admin_cq_init(ena_t *ena)
{
        ena_adminq_t *aq = &ena->ena_aq;
        ena_dma_buf_t *dma = &aq->ea_cq.eac_dma;
        uint32_t addr_low, addr_high, wval;

        if (aq->ea_cq.eac_entries == NULL) {
                size_t size = aq->ea_qlen * sizeof (*aq->ea_cq.eac_entries);
                ena_dma_conf_t conf = {
                        .edc_size = size,
                        .edc_align = ENAHW_ADMIN_CQ_DESC_BUF_ALIGNMENT,
                        .edc_sgl = 1,
                        .edc_endian = DDI_NEVERSWAP_ACC,
                        .edc_stream = false,
                };

                if (!ena_dma_alloc(ena, dma, &conf, size)) {
                        ena_err(ena, "failed to allocate DMA for Admin CQ");
                        return (false);
                }

                ENA_DMA_VERIFY_ADDR(ena, dma->edb_cookie->dmac_laddress);
                aq->ea_cq.eac_entries = (void *)dma->edb_va;
        } else {
                ena_dma_bzero(dma);
        }

        aq->ea_cq.eac_head = 0;
        aq->ea_cq.eac_phase = 1;
        addr_low = (uint32_t)(dma->edb_cookie->dmac_laddress);
        addr_high = (uint32_t)(dma->edb_cookie->dmac_laddress >> 32);
        ena_hw_bar_write32(ena, ENAHW_REG_ACQ_BASE_LO, addr_low);
        ena_hw_bar_write32(ena, ENAHW_REG_ACQ_BASE_HI, addr_high);
        wval = ENAHW_ACQ_CAPS_DEPTH(aq->ea_qlen) |
            ENAHW_ACQ_CAPS_ENTRY_SIZE(sizeof (*aq->ea_cq.eac_entries));
        ena_hw_bar_write32(ena, ENAHW_REG_ACQ_CAPS, wval);

        return (true);
}

void
ena_update_hints(ena_t *ena, enahw_device_hints_t *hints)
{
        ena->ena_device_hints.eh_mmio_read_timeout =
            hints->edh_mmio_read_timeout;
        ena->ena_device_hints.eh_keep_alive_timeout =
            hints->edh_keep_alive_timeout;
        ena->ena_device_hints.eh_tx_comp_timeout = hints->edh_tx_comp_timeout;
        ena->ena_device_hints.eh_missed_tx_reset_threshold =
            hints->edh_missed_tx_reset_threshold;
        ena->ena_device_hints.eh_admin_comp_timeout =
            hints->edh_admin_comp_timeout;
        ena->ena_device_hints.eh_max_tx_sgl = hints->edh_max_tx_sgl;
        ena->ena_device_hints.eh_max_rx_sgl = hints->edh_max_rx_sgl;
}

/*
 * We limit the max number of I/O queues based on several aspects of
 * the underlying hardware.
 *
 * 1. The absolute upper limit is set by ENAHW_MAX_NUM_IO_QUEUES,
 *    which comes from the common code and presumably is based on device
 *    constraints.
 *
 * 2. Next we latch the number of I/O queues to the number of online
 *    CPUs. The idea being that each queue is a parallel work stream,
 *    and having more queues than CPUs to flush them will not improve
 *    performance. The number of online CPUs can change dynamically,
 *    and that's okay, everything should still work fine, it just
 *    might not be ideal.
 *
 * 3. Next we latch the number of I/O queues to the smallest of the
 *    max Tx queues and max Rx queues. We could probably loosen this
 *    restriction in the future, and have separate max I/O queues for
 *    Tx and Rx. This is what Linux does, and seems like a fine place
 *    to start.
 */
static void
ena_set_max_io_queues(ena_t *ena)
{
        uint32_t max = ENAHW_MAX_NUM_IO_QUEUES;

        max = MIN(ncpus_online, max);
        /*
         * Supposedly a device could present a different number of SQs
         * and CQs. This driver is designed in a way that requires
         * each SQ to have a corresponding and dedicated CQ (how would
         * it work otherwise). Therefore, we must check both values
         * and find the minimum between them.
         */
        max = MIN(ena->ena_tx_max_sq_num, max);
        max = MIN(ena->ena_tx_max_cq_num, max);
        max = MIN(ena->ena_rx_max_sq_num, max);
        max = MIN(ena->ena_rx_max_cq_num, max);


        /* This shouldn't happen, but just in case. */
        if (max == 0) {
                max = 1;
        }

        ena->ena_max_io_queues = max;
}

/*
 * We require that an Rx or Tx buffer be able to hold the maximum MTU
 * along with the maximum frame header length. In this case we know
 * ENA is presenting us an Ethernet frame so we add the size of an
 * Ethernet VLAN header. Rx has the additional requirement of needing
 * additional margin for the sake of IP header alignment.
 */
static void
ena_update_buf_sizes(ena_t *ena)
{
        ena->ena_max_frame_hdr = sizeof (struct ether_vlan_header);
        ena->ena_max_frame_total = ena->ena_max_frame_hdr + ena->ena_mtu;
        ena->ena_tx_buf_sz = P2ROUNDUP_TYPED(ena->ena_max_frame_total,
            ena->ena_page_sz, uint32_t);
        ena->ena_rx_buf_sz = P2ROUNDUP_TYPED(ena->ena_max_frame_total +
            ENA_RX_BUF_IPHDR_ALIGNMENT, ena->ena_page_sz, uint32_t);
}

static bool
ena_get_hints(ena_t *ena)
{
        int ret;
        enahw_resp_desc_t resp;
        enahw_device_hints_t *hints = &resp.erd_resp.erd_get_feat.ergf_hints;

        ena_dbg(ena, "Requesting hints");

        bzero(&resp, sizeof (resp));
        ret = ena_get_feature(ena, &resp, ENAHW_FEAT_HW_HINTS,
            ENAHW_FEAT_HW_HINTS_VER);

        if (ret == ENOTSUP) {
                /* In this case the device does not support querying hints */
                ena_dbg(ena, "Hints are unsupported");
                return (true);
        } else if (ret != 0) {
                ena_err(ena, "Error getting hints: %d", ret);
                return (false);
        }

        ena_update_hints(ena, hints);

        return (true);
}

static bool
ena_get_offloads(ena_t *ena)
{
        int ret = 0;
        enahw_resp_desc_t resp;
        enahw_feat_offload_t *feat = &resp.erd_resp.erd_get_feat.ergf_offload;

        ena->ena_tx_l3_ipv4_csum = false;

        ena->ena_tx_l4_ipv4_part_csum = false;
        ena->ena_tx_l4_ipv4_full_csum = false;
        ena->ena_tx_l4_ipv4_lso = false;

        ena->ena_tx_l4_ipv6_part_csum = false;
        ena->ena_tx_l4_ipv6_full_csum = false;
        ena->ena_tx_l4_ipv6_lso = false;

        ena->ena_rx_l3_ipv4_csum = false;
        ena->ena_rx_l4_ipv4_csum = false;
        ena->ena_rx_l4_ipv6_csum = false;
        ena->ena_rx_hash = false;

        bzero(&resp, sizeof (resp));
        ret = ena_get_feature(ena, &resp, ENAHW_FEAT_STATELESS_OFFLOAD_CONFIG,
            ENAHW_FEAT_STATELESS_OFFLOAD_CONFIG_VER);

        if (ret == ENOTSUP) {
                /*
                 * In this case the device does not support querying
                 * for hardware offloads. We take that as a sign that
                 * the device provides no offloads.
                 */
                return (true);
        } else if (ret != 0) {
                ena_err(ena, "error getting stateless offload: %d", ret);
                return (false);
        }

        ena->ena_tx_l3_ipv4_csum = ENAHW_FEAT_OFFLOAD_TX_L3_IPV4_CSUM(feat);

        ena->ena_tx_l4_ipv4_part_csum =
            ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_PART(feat);
        ena->ena_tx_l4_ipv4_full_csum =
            ENAHW_FEAT_OFFLOAD_TX_L4_IPV4_CSUM_FULL(feat);
        ena->ena_tx_l4_ipv4_lso = ENAHW_FEAT_OFFLOAD_TSO_IPV4(feat);

        ena->ena_tx_l4_ipv6_part_csum =
            ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_PART(feat);
        ena->ena_tx_l4_ipv6_full_csum =
            ENAHW_FEAT_OFFLOAD_TX_L4_IPV6_CSUM_FULL(feat);
        ena->ena_tx_l4_ipv6_lso = ENAHW_FEAT_OFFLOAD_TSO_IPV6(feat);

        ena->ena_rx_l3_ipv4_csum = ENAHW_FEAT_OFFLOAD_RX_L3_IPV4_CSUM(feat);
        ena->ena_rx_l4_ipv4_csum = ENAHW_FEAT_OFFLOAD_RX_L4_IPV4_CSUM(feat);
        ena->ena_rx_l4_ipv6_csum = ENAHW_FEAT_OFFLOAD_RX_L4_IPV6_CSUM(feat);
        return (true);
}

static int
ena_get_prop(ena_t *ena, char *propname, const int minval, const int maxval,
    const int defval)
{
        int value = ddi_prop_get_int(DDI_DEV_T_ANY, ena->ena_dip,
            DDI_PROP_DONTPASS, propname, defval);

        if (value > maxval) {
                ena_err(ena, "user value %s=%d exceeded maximum, setting to %d",
                    propname, value, maxval);
                value = maxval;
        }

        if (value < minval) {
                ena_err(ena, "user value %s=%d below minimum, setting to %d",
                    propname, value, minval);
                value = minval;
        }

        return (value);
}

static bool
ena_set_mtu(ena_t *ena)
{
        int ret = 0;
        enahw_cmd_desc_t cmd;
        enahw_feat_mtu_t *feat = &cmd.ecd_cmd.ecd_set_feat.ecsf_feat.ecsf_mtu;
        enahw_resp_desc_t resp;

        bzero(&cmd, sizeof (cmd));
        bzero(&resp, sizeof (resp));
        feat->efm_mtu = ena->ena_mtu;

        if ((ret = ena_set_feature(ena, &cmd, &resp, ENAHW_FEAT_MTU,
            ENAHW_FEAT_MTU_VER)) != 0) {
                ena_err(ena, "failed to set device MTU to %u: %d", ena->ena_mtu,
                    ret);
                return (false);
        }

        return (true);
}

static void
ena_get_link_config(ena_t *ena)
{
        enahw_resp_desc_t resp;
        enahw_feat_link_conf_t *feat =
            &resp.erd_resp.erd_get_feat.ergf_link_conf;
        bool full_duplex;

        bzero(&resp, sizeof (resp));

        if (ena_get_feature(ena, &resp, ENAHW_FEAT_LINK_CONFIG,
            ENAHW_FEAT_LINK_CONFIG_VER) != 0) {
                /*
                 * Some ENA devices do not support this feature. In
                 * those cases we report a 1Gbps link, full duplex.
                 * For the most accurate information on bandwidth
                 * limits see the official AWS documentation.
                 */
                ena->ena_link_speed_mbits = 1000;
                ena->ena_link_speeds = ENAHW_LINK_SPEED_1G;
                ena->ena_link_duplex = LINK_DUPLEX_FULL;
                ena->ena_link_autoneg = true;
                return;
        }

        ena->ena_link_speed_mbits = feat->eflc_speed;
        ena->ena_link_speeds = feat->eflc_supported;
        full_duplex = ENAHW_FEAT_LINK_CONF_FULL_DUPLEX(feat);
        ena->ena_link_duplex = full_duplex ? LINK_DUPLEX_FULL :
            LINK_DUPLEX_HALF;
        ena->ena_link_autoneg = ENAHW_FEAT_LINK_CONF_AUTONEG(feat);
}

/*
 * Retrieve all configuration values which are modifiable via
 * ena.conf, and set ena_t members accordingly. While the conf values
 * have priority, they may be implicitly modified by the driver to
 * meet resource constraints on a given platform. If no value is
 * specified in the conf file, the driver will attempt to use the
 * largest value supported. While there should be no value large
 * enough, keep in mind that ena_get_prop() will cast the values to an
 * int.
 *
 * This function should be called after the device is initialized,
 * admin queue is established, and the hardware features/capabs have
 * been queried; it should be called before mac registration.
 */
static bool
ena_attach_read_conf(ena_t *ena)
{
        uint32_t gcv;   /* Greatest Common Value */

        /*
         * We expect that the queue lengths are the same for both the
         * CQ and SQ, but technically the device could return
         * different lengths. For now the driver locks them together.
         */
        gcv = MIN(ena->ena_rx_max_sq_num_descs, ena->ena_rx_max_cq_num_descs);
        ASSERT3U(gcv, <=, INT_MAX);
        ena->ena_rxq_num_descs = ena_get_prop(ena, ENA_PROP_RXQ_NUM_DESCS,
            ENA_PROP_RXQ_NUM_DESCS_MIN, gcv, gcv);

        ena->ena_rxq_intr_limit = ena_get_prop(ena, ENA_PROP_RXQ_INTR_LIMIT,
            ENA_PROP_RXQ_INTR_LIMIT_MIN, ENA_PROP_RXQ_INTR_LIMIT_MAX,
            ENA_PROP_RXQ_INTR_LIMIT_DEF);

        gcv = MIN(ena->ena_tx_max_sq_num_descs, ena->ena_tx_max_cq_num_descs);
        ASSERT3U(gcv, <=, INT_MAX);
        ena->ena_txq_num_descs = ena_get_prop(ena, ENA_PROP_TXQ_NUM_DESCS,
            ENA_PROP_TXQ_NUM_DESCS_MIN, gcv, gcv);

        return (true);
}

/*
 * Perform any necessary device configuration after the driver.conf
 * has been read.
 */
static bool
ena_attach_dev_cfg(ena_t *ena)
{
        ASSERT3U(ena->ena_attach_seq, >=, ENA_ATTACH_READ_CONF);

        if (!ena_set_mtu(ena)) {
                /*
                 * We don't expect this to fail, but we try a fallback
                 * first before failing the attach sequence.
                 */
                ena->ena_mtu = 1500;
                ena_err(ena, "trying fallback MTU: %u", ena->ena_mtu);

                if (!ena_set_mtu(ena)) {
                        return (false);
                }
        }

        return (true);
}

static bool
ena_check_versions(ena_t *ena)
{
        uint32_t dev_vsn = ena_hw_bar_read32(ena, ENAHW_REG_VERSION);
        uint32_t ctrl_vsn =
            ena_hw_bar_read32(ena, ENAHW_REG_CONTROLLER_VERSION);

        ena->ena_dev_major_vsn = ENAHW_DEV_MAJOR_VSN(dev_vsn);
        ena->ena_dev_minor_vsn = ENAHW_DEV_MINOR_VSN(dev_vsn);

        ena->ena_ctrl_major_vsn = ENAHW_CTRL_MAJOR_VSN(ctrl_vsn);
        ena->ena_ctrl_minor_vsn = ENAHW_CTRL_MINOR_VSN(ctrl_vsn);
        ena->ena_ctrl_subminor_vsn = ENAHW_CTRL_SUBMINOR_VSN(ctrl_vsn);
        ena->ena_ctrl_impl_id = ENAHW_CTRL_IMPL_ID(ctrl_vsn);

        ena_dbg(ena, "device version: %u.%u",
            ena->ena_dev_major_vsn, ena->ena_dev_minor_vsn);
        ena_dbg(ena, "controller version: %u.%u.%u implementation %u",
            ena->ena_ctrl_major_vsn, ena->ena_ctrl_minor_vsn,
            ena->ena_ctrl_subminor_vsn, ena->ena_ctrl_impl_id);

        if (ena->ena_ctrl_subminor_vsn < ENA_CTRL_SUBMINOR_VSN_MIN) {
                ena_err(ena, "unsupported controller version: %u.%u.%u",
                    ena->ena_ctrl_major_vsn, ena->ena_ctrl_minor_vsn,
                    ena->ena_ctrl_subminor_vsn);
                return (false);
        }

        return (true);
}

static bool
ena_adminq_init(ena_t *ena)
{
        ena_adminq_t *aq = &ena->ena_aq;

        /*
         * As we are not using an interrupt for admin queue completion
         * signaling, we do not need a priority on these mutexes. If
         * that changes, we will have to rejigger some code to create
         * the admin queue interrupt before this function.
         */
        mutex_init(&aq->ea_sq_lock, NULL, MUTEX_DRIVER, NULL);
        mutex_init(&aq->ea_cq_lock, NULL, MUTEX_DRIVER, NULL);
        mutex_init(&aq->ea_stat_lock, NULL, MUTEX_DRIVER, NULL);
        aq->ea_qlen = ENA_ADMINQ_DEPTH;
        aq->ea_pending_cmds = 0;

        aq->ea_cmd_ctxs = kmem_zalloc(sizeof (ena_cmd_ctx_t) * aq->ea_qlen,
            KM_SLEEP);
        list_create(&aq->ea_cmd_ctxs_free, sizeof (ena_cmd_ctx_t),
            offsetof(ena_cmd_ctx_t, ectx_node));
        list_create(&aq->ea_cmd_ctxs_used, sizeof (ena_cmd_ctx_t),
            offsetof(ena_cmd_ctx_t, ectx_node));

        ena_create_cmd_ctx(ena);

        /*
         * Start in polling mode until we've determined the number of queues
         * and are ready to configure and enable interrupts.
         */
        ena_hw_bar_write32(ena, ENAHW_REG_INTERRUPT_MASK, ENAHW_INTR_MASK);
        aq->ea_poll_mode = true;

        return (true);
}

/*
 * Free all resources allocated as part of ena_device_init().
 */
static void
ena_cleanup_device_init(ena_t *ena, bool resetting)
{
        ena_adminq_t *aq = &ena->ena_aq;

        VERIFY0(resetting);

        if (ena->ena_llq_bar_mapped) {
                ddi_regs_map_free(&ena->ena_llq_bar_hdl);
                ena->ena_llq_bar_mapped = false;
        }

        ena_free_host_info(ena);
        mutex_destroy(&aq->ea_sq_lock);
        mutex_destroy(&aq->ea_cq_lock);
        mutex_destroy(&aq->ea_stat_lock);
        list_destroy(&aq->ea_cmd_ctxs_free);
        list_destroy(&aq->ea_cmd_ctxs_used);
        kmem_free(aq->ea_cmd_ctxs, sizeof (ena_cmd_ctx_t) * aq->ea_qlen);
        ena_admin_sq_free(ena);
        ena_admin_cq_free(ena);
        ena_aenq_free(ena);
        ena_stat_device_cleanup(ena);
        ena_stat_device_basic_cleanup(ena);
        ena_stat_device_extended_cleanup(ena);
        ena_stat_aenq_cleanup(ena);
}

/*
 * We explicitly disable hardware timestamping as a precaution.
 * When enabled, the device uses (larger) extended completion descriptors to
 * incorporate timestamp fields. Our completion processing assumes the base
 * descriptor sizes.
 */
static bool
ena_disable_hw_timestamp(ena_t *ena)
{
        enahw_cmd_desc_t cmd;
        enahw_resp_desc_t resp;
        enahw_feat_hw_timestamp_t *ts =
            &cmd.ecd_cmd.ecd_set_feat.ecsf_feat.ecsf_hw_timestamp;
        int ret;

        if (!ena_is_feat_avail(ena, ENAHW_FEAT_HW_TIMESTAMP))
                return (true);

        ena_dbg(ena, "Disabling HW timestamping");

        bzero(&cmd, sizeof (cmd));
        ts->efhwts_tx = ENAHW_HW_TIMESTAMP_NONE;
        ts->efhwts_rx = ENAHW_HW_TIMESTAMP_NONE;

        ret = ena_set_feature(ena, &cmd, &resp, ENAHW_FEAT_HW_TIMESTAMP,
            ENAHW_FEAT_HW_TIMESTAMP_VER);
        if (ret != 0) {
                ena_err(ena, "failed to disable HW timestamping: %d", ret);
                return (false);
        }

        return (true);
}

/*
 * Map the device memory BAR used for LLQ. This must be called before creating
 * any LLQ submission queues.
 */
static bool
ena_map_llq_mem_bar(ena_t *ena)
{
        ddi_device_acc_attr_t attr;
        int rnumber, ret;

        rnumber = ena_bar_to_rnumber(ena, ENA_LLQ_BAR);
        if (rnumber == -1) {
                ena_err(ena, "failed to find rnumber for BAR %d", ENA_LLQ_BAR);
                return (false);
        }

        if (ddi_dev_regsize(ena->ena_dip, rnumber, &ena->ena_llq_bar_size) !=
            DDI_SUCCESS) {
                ena_err(ena, "failed to get mem BAR %d size", ENA_LLQ_BAR);
                return (false);
        }

        bzero(&attr, sizeof (attr));
        attr.devacc_attr_version = DDI_DEVICE_ATTR_V1;
        attr.devacc_attr_endian_flags = DDI_NEVERSWAP_ACC;
        /*
         * See the "LLQ Tx Path" section of the big theory statement for why
         * it is safe to map the LLQ BAR with DDI_STORECACHING_OK_ACC.
         */
        attr.devacc_attr_dataorder = DDI_STORECACHING_OK_ACC;
        attr.devacc_attr_access = DDI_DEFAULT_ACC;

        ret = ddi_regs_map_setup(ena->ena_dip, rnumber,
            &ena->ena_llq_bar_base, 0, ena->ena_llq_bar_size, &attr,
            &ena->ena_llq_bar_hdl);

        if (ret != DDI_SUCCESS) {
                ena_err(ena, "failed to map mem BAR %d (reg %d): %d",
                    ENA_LLQ_BAR, rnumber, ret);
                return (false);
        }

        ena->ena_llq_bar_mapped = true;
        ena_dbg(ena, "LLQ BAR %d mapped to base: 0x%p size: %ld", ENA_LLQ_BAR,
            (void *)ena->ena_llq_bar_base, ena->ena_llq_bar_size);

        return (true);
}

/*
 * Negotiate LLQ parameters with the device and send SET_FEATURE to
 * configure it. If the device does not advertise LLQ support we
 * silently fall back to host placement. If the device does advertise
 * LLQ but negotiation fails, we treat that as a fatal error rather
 * than falling back. For newer instance types (Nitro v5 and later)
 * it really is fatal, they require LLQ and do not support host
 * placement at all.
 */
static bool
ena_configure_llq(ena_t *ena)
{
        enahw_resp_desc_t resp;
        enahw_feat_llq_t *llq_feat;
        enahw_cmd_desc_t cmd;
        enahw_feat_llq_t *llq_cmd;
        enahw_llq_accel_mode_get_t accel_get;
        uint16_t supported;
        int ret;

        /* Query the device's LLQ capabilities */
        bzero(&resp, sizeof (resp));
        ret = ena_get_feature(ena, &resp, ENAHW_FEAT_LLQ, ENAHW_FEAT_LLQ_VER);
        if (ret == ENOTSUP) {
                ena_dbg(ena, "LLQ not supported, using host placement");
                return (true);
        } else if (ret != 0) {
                ena_err(ena, "failed to query LLQ feature: %d", ret);
                return (false);
        }

        llq_feat = &resp.erd_resp.erd_get_feat.ergf_llq;

        ena_dbg(ena, "LLQ max_num=%u max_depth=%u",
            llq_feat->efllq_max_llq_num, llq_feat->efllq_max_llq_depth);
        ena_dbg(ena, "LLQ header_location supported=0x%x "
            "entry_size supported=0x%x recommended=%u",
            llq_feat->efllq_header_location_ctrl_supported,
            llq_feat->efllq_entry_size_ctrl_supported,
            llq_feat->efllq_entry_size_recommended);
        ena_dbg(ena, "LLQ descs_before_header supported=0x%x "
            "stride supported=0x%x",
            llq_feat->efllq_desc_num_before_header_supported,
            llq_feat->efllq_descs_stride_ctrl_supported);
        ena_dbg(ena, "LLQ accel_mode supported_flags=0x%x "
            "max_tx_burst_size=%u",
            llq_feat->efllq_accel_mode.eam_get.eamg_supported_flags,
            llq_feat->efllq_accel_mode.eam_get.eamg_max_tx_burst_size);

        /*
         * Negotiate header location - we currently only support inline header,
         * which is advertised by all current devices.
         */
        supported = llq_feat->efllq_header_location_ctrl_supported;
        if (!(supported & ENAHW_LLQ_HEADER_INLINE)) {
                ena_err(ena, "device does not support inline header "
                    "LLQ (supported: 0x%x)", supported);
                return (false);
        }
        ena->ena_llq_header_location = ENAHW_LLQ_HEADER_INLINE;

        /* Negotiate stride control for inline header mode */
        supported = llq_feat->efllq_descs_stride_ctrl_supported;
        if (supported & ENAHW_LLQ_MULTIPLE_DESCS_PER_ENTRY) {
                ena->ena_llq_stride_ctrl = ENAHW_LLQ_MULTIPLE_DESCS_PER_ENTRY;
        } else if (supported & ENAHW_LLQ_SINGLE_DESC_PER_ENTRY) {
                ena->ena_llq_stride_ctrl = ENAHW_LLQ_SINGLE_DESC_PER_ENTRY;
        } else {
                ena_err(ena, "no supported LLQ stride control (0x%x)",
                    supported);
                return (false);
        }

        /* Negotiate entry size */
        supported = llq_feat->efllq_entry_size_ctrl_supported;
        if (llq_feat->efllq_entry_size_recommended != 0 &&
            (supported & llq_feat->efllq_entry_size_recommended)) {
                ena->ena_llq_entry_size =
                    llq_feat->efllq_entry_size_recommended;
        } else if (supported & ENAHW_LLQ_ENTRY_SIZE_128B) {
                ena->ena_llq_entry_size = ENAHW_LLQ_ENTRY_SIZE_128B;
        } else if (supported & ENAHW_LLQ_ENTRY_SIZE_192B) {
                ena->ena_llq_entry_size = ENAHW_LLQ_ENTRY_SIZE_192B;
        } else if (supported & ENAHW_LLQ_ENTRY_SIZE_256B) {
                ena->ena_llq_entry_size = ENAHW_LLQ_ENTRY_SIZE_256B;
        } else {
                ena_err(ena, "no supported LLQ entry size (0x%x)", supported);
                return (false);
        }

        switch (ena->ena_llq_entry_size) {
        case ENAHW_LLQ_ENTRY_SIZE_128B:
                ena->ena_llq_entry_size_bytes = 128;
                break;
        case ENAHW_LLQ_ENTRY_SIZE_192B:
                ena->ena_llq_entry_size_bytes = 192;
                break;
        case ENAHW_LLQ_ENTRY_SIZE_256B:
                ena->ena_llq_entry_size_bytes = 256;
                break;
        }

        /*
         * Negotiate number of descriptors before header. Since we always
         * send both a meta and data descriptor in each LLQ entry we cannot
         * support ENAHW_LLQ_NUM_DESCS_BEFORE_HEADER_1.
         */
        supported = llq_feat->efllq_desc_num_before_header_supported;
        if (supported & ENAHW_LLQ_NUM_DESCS_BEFORE_HEADER_2) {
                ena->ena_llq_num_descs_before_header =
                    ENAHW_LLQ_NUM_DESCS_BEFORE_HEADER_2;
        } else if (supported & ENAHW_LLQ_NUM_DESCS_BEFORE_HEADER_4) {
                ena->ena_llq_num_descs_before_header =
                    ENAHW_LLQ_NUM_DESCS_BEFORE_HEADER_4;
        } else if (supported & ENAHW_LLQ_NUM_DESCS_BEFORE_HEADER_8) {
                ena->ena_llq_num_descs_before_header =
                    ENAHW_LLQ_NUM_DESCS_BEFORE_HEADER_8;
        } else {
                ena_err(ena, "no supported descs_before_header (0x%x)",
                    supported);
                return (false);
        }

        /* Check for accelerated LLQ mode support */
        accel_get = llq_feat->efllq_accel_mode.eam_get;
        const bool limit_tx_burst = (accel_get.eamg_supported_flags &
            ENAHW_LLQ_ACCEL_MODE_LIMIT_TX_BURST);

        if (limit_tx_burst) {
                ena->ena_llq_max_tx_burst_size =
                    accel_get.eamg_max_tx_burst_size;
        }

        ena_dbg(ena, "LLQ negotiated: header_location=%u entry_size=%uB "
            "descs_before_header=%u stride=%u max_tx_burst=%u",
            ena->ena_llq_header_location,
            ena->ena_llq_entry_size_bytes,
            ena->ena_llq_num_descs_before_header,
            ena->ena_llq_stride_ctrl,
            ena->ena_llq_max_tx_burst_size);

        /*
         * When LLQ is active the SQ lives in device memory rather
         * than host memory so max_tx_sq_depth does not apply;
         * replace it with max_llq_depth. For 256B entries the depth
         * is further reduced; the device may report a specific wide
         * depth, otherwise we halve the standard LLQ depth.
         */
        ena->ena_tx_max_sq_num_descs = llq_feat->efllq_max_llq_depth;

        if (ena->ena_llq_entry_size == ENAHW_LLQ_ENTRY_SIZE_256B) {
                if (llq_feat->efllq_max_wide_llq_depth != 0) {
                        ena->ena_tx_max_sq_num_descs = MIN(
                            ena->ena_tx_max_sq_num_descs,
                            llq_feat->efllq_max_wide_llq_depth);
                } else {
                        ena->ena_tx_max_sq_num_descs /= 2;
                }
        }

        /* Send the LLQ configuration request */
        bzero(&cmd, sizeof (cmd));
        llq_cmd = &cmd.ecd_cmd.ecd_set_feat.ecsf_feat.ecsf_llq;

        llq_cmd->efllq_header_location_ctrl_enabled =
            ena->ena_llq_header_location;
        llq_cmd->efllq_entry_size_ctrl_enabled = ena->ena_llq_entry_size;
        llq_cmd->efllq_desc_num_before_header_enabled =
            ena->ena_llq_num_descs_before_header;
        llq_cmd->efllq_descs_stride_ctrl_enabled = ena->ena_llq_stride_ctrl;
        llq_cmd->efllq_accel_mode.eam_set.eams_enabled_flags =
            ENAHW_LLQ_ACCEL_MODE_DISABLE_META_CACHING |
            (limit_tx_burst ? ENAHW_LLQ_ACCEL_MODE_LIMIT_TX_BURST : 0);

        bzero(&resp, sizeof (resp));
        ret = ena_set_feature(ena, &cmd, &resp, ENAHW_FEAT_LLQ,
            ENAHW_FEAT_LLQ_VER);
        if (ret != 0) {
                ena_err(ena, "failed to set LLQ feature: %d", ret);
                return (false);
        }

        if (!ena_map_llq_mem_bar(ena))
                return (false);

        ena->ena_llq_enabled = true;
        ena_dbg(ena, "LLQ enabled");

        return (true);
}

static bool
ena_attach_device_init(ena_t *ena)
{
        ena_adminq_t *aq = &ena->ena_aq;
        uint32_t rval;
        uint8_t dma_width;
        hrtime_t cmd_timeout;
        enahw_resp_desc_t resp;
        enahw_feat_dev_attr_t *feat = &resp.erd_resp.erd_get_feat.ergf_dev_attr;
        uint8_t *maddr;
        uint32_t supported_features;
        int ret = 0;

        ena->ena_reset_reason = ENAHW_RESET_NORMAL;
        if (!ena_device_reset(ena, ena->ena_reset_reason))
                return (false);

        if (!ena_check_versions(ena))
                return (false);

        ena_init_regcache(ena);

        rval = ena_hw_bar_read32(ena, ENAHW_REG_CAPS);
        dma_width = ENAHW_CAPS_DMA_ADDR_WIDTH(rval);
        ena->ena_dma_width = dma_width;

        /*
         * The value stored in the device register is in the
         * resolution of 100 milliseconds. We normalize that to
         * nanoseconds.
         */
        cmd_timeout = MSEC2NSEC(ENAHW_CAPS_ADMIN_CMD_TIMEOUT(rval) * 100);
        aq->ea_cmd_timeout_ns = max(cmd_timeout, ena_admin_cmd_timeout_ns);

        if (aq->ea_cmd_timeout_ns == 0)
                aq->ea_cmd_timeout_ns = ENA_ADMIN_CMD_DEF_TIMEOUT_NS;

        if (!ena_adminq_init(ena))
                return (false);

        if (!ena_admin_sq_init(ena))
                return (false);

        if (!ena_admin_cq_init(ena))
                return (false);

        if (!ena_aenq_init(ena))
                return (false);

        /*
         * The device will change the set of advertised features based on the
         * host info that we provide. We therefore need to send this here
         * before querying the features.
         */
        if (!ena_init_host_info(ena))
                return (false);

        bzero(&resp, sizeof (resp));
        ret = ena_get_feature(ena, &resp, ENAHW_FEAT_DEVICE_ATTRIBUTES,
            ENAHW_FEAT_DEVICE_ATTRIBUTES_VER);

        if (ret != 0) {
                ena_err(ena, "failed to get device attributes: %d", ret);
                return (false);
        }

        ena_dbg(ena, "impl ID: %u", feat->efda_impl_id);
        ena_dbg(ena, "device version: %u", feat->efda_device_version);
        ena_dbg(ena, "supported features: 0x%x",
            feat->efda_supported_features);
        ena_dbg(ena, "device capabilities: 0x%x", feat->efda_capabilities);
        ena_dbg(ena, "phys addr width: %u", feat->efda_phys_addr_width);
        ena_dbg(ena, "virt addr width: %u", feat->efda_virt_addr_with);
        maddr = feat->efda_mac_addr;
        ena_dbg(ena, "mac addr: %x:%x:%x:%x:%x:%x", maddr[0], maddr[1],
            maddr[2], maddr[3], maddr[4], maddr[5]);
        ena_dbg(ena, "max MTU: %u", feat->efda_max_mtu);

        bcopy(maddr, ena->ena_mac_addr, ETHERADDRL);
        ena->ena_max_mtu = feat->efda_max_mtu;
        ena->ena_capabilities = feat->efda_capabilities;
        supported_features = feat->efda_supported_features;
        ena->ena_supported_features = supported_features;
        feat = NULL;
        bzero(&resp, sizeof (resp));

        if (ena_is_feat_avail(ena, ENAHW_FEAT_MAX_QUEUES_EXT)) {
                enahw_feat_max_queue_ext_t *feat_mqe =
                    &resp.erd_resp.erd_get_feat.ergf_max_queue_ext;

                ret = ena_get_feature(ena, &resp, ENAHW_FEAT_MAX_QUEUES_EXT,
                    ENAHW_FEAT_MAX_QUEUES_EXT_VER);

                if (ret != 0) {
                        ena_err(ena, "failed to query max queues ext: %d", ret);
                        return (false);
                }

                ena->ena_tx_max_sq_num = feat_mqe->efmqe_max_tx_sq_num;
                ena->ena_tx_max_sq_num_descs = feat_mqe->efmqe_max_tx_sq_depth;
                ena->ena_tx_max_cq_num = feat_mqe->efmqe_max_tx_cq_num;
                ena->ena_tx_max_cq_num_descs = feat_mqe->efmqe_max_tx_cq_depth;
                ena->ena_tx_max_desc_per_pkt =
                    feat_mqe->efmqe_max_per_packet_tx_descs;
                ena->ena_tx_max_hdr_len = feat_mqe->efmqe_max_tx_header_size;

                ena->ena_rx_max_sq_num = feat_mqe->efmqe_max_rx_sq_num;
                ena->ena_rx_max_sq_num_descs = feat_mqe->efmqe_max_rx_sq_depth;
                ena->ena_rx_max_cq_num = feat_mqe->efmqe_max_rx_cq_num;
                ena->ena_rx_max_cq_num_descs = feat_mqe->efmqe_max_rx_cq_depth;
                ena->ena_rx_max_desc_per_pkt =
                    feat_mqe->efmqe_max_per_packet_rx_descs;

                ena_set_max_io_queues(ena);
        } else {
                enahw_feat_max_queue_t *feat_mq =
                    &resp.erd_resp.erd_get_feat.ergf_max_queue;

                ret = ena_get_feature(ena, &resp, ENAHW_FEAT_MAX_QUEUES_NUM,
                    ENAHW_FEAT_MAX_QUEUES_NUM_VER);

                if (ret != 0) {
                        ena_err(ena, "failed to query max queues: %d", ret);
                        return (false);
                }

                ena->ena_tx_max_sq_num = feat_mq->efmq_max_sq_num;
                ena->ena_tx_max_sq_num_descs = feat_mq->efmq_max_sq_depth;
                ena->ena_tx_max_cq_num = feat_mq->efmq_max_cq_num;
                ena->ena_tx_max_cq_num_descs = feat_mq->efmq_max_cq_depth;
                ena->ena_tx_max_desc_per_pkt =
                    feat_mq->efmq_max_per_packet_tx_descs;
                ena->ena_tx_max_hdr_len = feat_mq->efmq_max_header_size;

                ena->ena_rx_max_sq_num = feat_mq->efmq_max_sq_num;
                ena->ena_rx_max_sq_num_descs = feat_mq->efmq_max_sq_depth;
                ena->ena_rx_max_cq_num = feat_mq->efmq_max_cq_num;
                ena->ena_rx_max_cq_num_descs = feat_mq->efmq_max_cq_depth;
                ena->ena_rx_max_desc_per_pkt =
                    feat_mq->efmq_max_per_packet_rx_descs;

                ena_set_max_io_queues(ena);
        }

        ena->ena_mtu = ena->ena_max_mtu;
        ena_update_buf_sizes(ena);

        if (!ena_get_hints(ena))
                return (false);

        ena->ena_tx_sgl_max_sz = 1;
        ena->ena_rx_sgl_max_sz = 1;
        if (ena->ena_device_hints.eh_max_tx_sgl != 0)
                ena->ena_tx_sgl_max_sz = ena->ena_device_hints.eh_max_tx_sgl;
        if (ena->ena_device_hints.eh_max_rx_sgl != 0)
                ena->ena_rx_sgl_max_sz = ena->ena_device_hints.eh_max_rx_sgl;

        if (!ena_disable_hw_timestamp(ena))
                return (false);

        if (!ena_configure_llq(ena))
                return (false);

        if (!ena_aenq_configure(ena))
                return (false);

        ena_get_link_config(ena);

        if (!ena_get_offloads(ena))
                return (false);

        if (!ena_stat_device_init(ena))
                return (false);

        if (!ena_stat_device_basic_init(ena))
                return (false);

        if (!ena_stat_device_extended_init(ena))
                return (false);

        if (!ena_stat_aenq_init(ena))
                return (false);

        ena_update_regcache(ena);

        return (true);
}

static void
ena_cleanup_intr_alloc(ena_t *ena, bool resetting)
{
        VERIFY0(resetting);

        for (int i = 0; i < ena->ena_num_intrs; i++) {
                int ret = ddi_intr_free(ena->ena_intr_handles[i]);
                if (ret != DDI_SUCCESS) {
                        ena_err(ena, "failed to free interrupt %d: %d", i, ret);
                }
        }

        if (ena->ena_intr_handles != NULL) {
                kmem_free(ena->ena_intr_handles, ena->ena_intr_handles_sz);
                ena->ena_intr_handles = NULL;
                ena->ena_intr_handles_sz = 0;
        }
}

/*
 * The Linux driver supports only MSI-X interrupts. We do the same,
 * with the assumption that it's the only type of interrupt the device
 * can present.
 */
static bool
ena_attach_intr_alloc(ena_t *ena)
{
        int ret;
        int types;
        int min, req, ideal, avail, actual;

        ret = ddi_intr_get_supported_types(ena->ena_dip, &types);
        if (ret != DDI_SUCCESS) {
                ena_err(ena, "failed to get interrupt types: %d", ret);
                return (false);
        }

        ena_dbg(ena, "supported interrupt types: 0x%x", types);
        if ((types & DDI_INTR_TYPE_MSIX) == 0) {
                ena_err(ena, "the ena driver only supports MSI-X interrupts");
                return (false);
        }

        /* One for I/O, one for adminq. */
        min = 2;
        ideal = ena->ena_max_io_queues + 1;
        ret = ddi_intr_get_nintrs(ena->ena_dip, DDI_INTR_TYPE_MSIX, &avail);
        if (ret != DDI_SUCCESS) {
                ena_err(ena, "failed to get number of MSI-X interrupts: %d",
                    ret);
                return (false);
        }

        if (avail < min) {
                ena_err(ena, "number of MSI-X interrupts is %d, but the driver "
                    "requires a minimum of %d", avail, min);
                return (false);
        }

        ena_dbg(ena, "%d MSI-X interrupts available", avail);

        ret = ddi_intr_get_navail(ena->ena_dip, DDI_INTR_TYPE_MSIX, &avail);
        if (ret != DDI_SUCCESS) {
                ena_err(ena, "failed to get available interrupts: %d", ret);
                return (false);
        }

        if (avail < min) {
                ena_err(ena, "number of available MSI-X interrupts is %d, "
                    "but the driver requires a minimum of %d", avail, min);
                return (false);
        }

        req = MIN(ideal, avail);
        ena->ena_intr_handles_sz = req * sizeof (ddi_intr_handle_t);
        ena->ena_intr_handles = kmem_zalloc(ena->ena_intr_handles_sz, KM_SLEEP);

        ret = ddi_intr_alloc(ena->ena_dip, ena->ena_intr_handles,
            DDI_INTR_TYPE_MSIX, 0, req, &actual, DDI_INTR_ALLOC_NORMAL);
        if (ret != DDI_SUCCESS) {
                ena_err(ena, "failed to allocate %d MSI-X interrupts: %d",
                    req, ret);
                return (false);
        }

        if (actual < min) {
                ena_err(ena, "number of allocated interrupts is %d, but the "
                    "driver requires a minimum of %d", actual, min);
                return (false);
        }

        ena->ena_num_intrs = actual;

        ret = ddi_intr_get_cap(ena->ena_intr_handles[0], &ena->ena_intr_caps);
        if (ret != DDI_SUCCESS) {
                ena_err(ena, "failed to get interrupt capability: %d", ret);
                return (false);
        }

        ret = ddi_intr_get_pri(ena->ena_intr_handles[0], &ena->ena_intr_pri);
        if (ret != DDI_SUCCESS) {
                ena_err(ena, "failed to get interrupt priority: %d", ret);
                return (false);
        }

        ena_dbg(ena, "MSI-X interrupts allocated: %d, cap: 0x%x, pri: %u",
            actual, ena->ena_intr_caps, ena->ena_intr_pri);

        /*
         * The ena_lock should not be held in the data path, but it is
         * held as part of the AENQ handler, which runs in interrupt
         * context. Therefore, we delayed the initialization of this
         * mutex until after the interrupts are allocated.
         */
        mutex_init(&ena->ena_lock, NULL, MUTEX_DRIVER,
            DDI_INTR_PRI(ena->ena_intr_pri));
        mutex_init(&ena->ena_watchdog_lock, NULL, MUTEX_DRIVER, NULL);

        return (true);
}

/*
 * Allocate the parent Rx queue structures. More importantly, this is
 * NOT allocating the queue descriptors or data buffers. Those are
 * allocated on demand as queues are started.
 */
static bool
ena_attach_alloc_rxqs(ena_t *ena)
{
        bool resetting = false;

        if (ena->ena_rxqs == NULL) {
                /*
                 * We rely on the interrupt priority for initializing the
                 * mutexes.
                 */
                VERIFY3U(ena->ena_attach_seq, >=, ENA_ATTACH_INTR_ALLOC);
                ena->ena_num_rxqs = ena->ena_num_intrs - 1;
                ASSERT3U(ena->ena_num_rxqs, >, 0);
                ena->ena_rxqs = kmem_zalloc(
                    ena->ena_num_rxqs * sizeof (*ena->ena_rxqs), KM_SLEEP);
        } else {
                resetting = true;
        }

        for (uint_t i = 0; i < ena->ena_num_rxqs; i++) {
                ena_rxq_t *rxq = &ena->ena_rxqs[i];

                rxq->er_rxqs_idx = i;
                /* The 0th vector is for Admin + AENQ. */
                rxq->er_intr_vector = i + 1;
                rxq->er_mrh = NULL;

                if (!resetting) {
                        mutex_init(&rxq->er_lock, NULL, MUTEX_DRIVER,
                            DDI_INTR_PRI(ena->ena_intr_pri));
                        mutex_init(&rxq->er_stat_lock, NULL, MUTEX_DRIVER,
                            DDI_INTR_PRI(ena->ena_intr_pri));
                }

                rxq->er_ena = ena;
                rxq->er_sq_num_descs = ena->ena_rxq_num_descs;
                rxq->er_cq_num_descs = ena->ena_rxq_num_descs;

                if (!ena_stat_rxq_init(rxq)) {
                        return (false);
                }

                if (!ena_alloc_rxq(rxq)) {
                        ena_stat_rxq_cleanup(rxq);
                        return (false);
                }
        }

        return (true);
}

static void
ena_cleanup_rxqs(ena_t *ena, bool resetting)
{
        for (uint_t i = 0; i < ena->ena_num_rxqs; i++) {
                ena_rxq_t *rxq = &ena->ena_rxqs[i];

                ena_cleanup_rxq(rxq, resetting);
                if (!resetting) {
                        mutex_destroy(&rxq->er_lock);
                        mutex_destroy(&rxq->er_stat_lock);
                }
                ena_stat_rxq_cleanup(rxq);
        }

        if (!resetting) {
                kmem_free(ena->ena_rxqs,
                    ena->ena_num_rxqs * sizeof (*ena->ena_rxqs));
                ena->ena_rxqs = NULL;
        }
}

/*
 * Allocate the parent Tx queue structures. More importantly, this is
 * NOT allocating the queue descriptors or data buffers. Those are
 * allocated on demand as a queue is started.
 */
static bool
ena_attach_alloc_txqs(ena_t *ena)
{
        bool resetting = false;

        if (ena->ena_txqs == NULL) {
                /*
                 * We rely on the interrupt priority for initializing the
                 * mutexes.
                 */
                VERIFY3U(ena->ena_attach_seq, >=, ENA_ATTACH_INTR_ALLOC);
                ena->ena_num_txqs = ena->ena_num_intrs - 1;
                ASSERT3U(ena->ena_num_txqs, >, 0);
                ena->ena_txqs = kmem_zalloc(
                    ena->ena_num_txqs * sizeof (*ena->ena_txqs), KM_SLEEP);
        } else {
                resetting = true;
        }

        for (uint_t i = 0; i < ena->ena_num_txqs; i++) {
                ena_txq_t *txq = &ena->ena_txqs[i];

                txq->et_txqs_idx = i;
                /* The 0th vector is for Admin + AENQ. */
                txq->et_intr_vector = i + 1;
                txq->et_mrh = NULL;

                if (!resetting) {
                        mutex_init(&txq->et_lock, NULL, MUTEX_DRIVER,
                            DDI_INTR_PRI(ena->ena_intr_pri));
                        mutex_init(&txq->et_stat_lock, NULL, MUTEX_DRIVER,
                            DDI_INTR_PRI(ena->ena_intr_pri));
                }

                txq->et_ena = ena;
                txq->et_sq_num_descs = ena->ena_txq_num_descs;
                txq->et_cq_num_descs = ena->ena_txq_num_descs;

                if (!ena_stat_txq_init(txq)) {
                        return (false);
                }

                if (!ena_alloc_txq(txq)) {
                        ena_stat_txq_cleanup(txq);
                        return (false);
                }
        }

        return (true);
}

static void
ena_cleanup_txqs(ena_t *ena, bool resetting)
{
        for (uint_t i = 0; i < ena->ena_num_txqs; i++) {
                ena_txq_t *txq = &ena->ena_txqs[i];

                ena_cleanup_txq(txq, resetting);
                if (!resetting) {
                        mutex_destroy(&txq->et_lock);
                        mutex_destroy(&txq->et_stat_lock);
                }
                ena_stat_txq_cleanup(txq);
        }

        if (!resetting) {
                kmem_free(ena->ena_txqs,
                    ena->ena_num_txqs * sizeof (*ena->ena_txqs));
                ena->ena_txqs = NULL;
        }
}

/*
 * To reset the device we need to unwind some of the steps taken during attach
 * but, since the device could well be in a failed state, we cannot rely on
 * being able to talk via the admin queue to do things such as explicitly
 * destroy rings. We call selected cleanup handlers with the second parameter
 * set to "true" to indicate that we are resetting and should avoid such
 * communication.
 *
 * The existing DMA memory regions for the admin queue, async event queue and
 * host information are preserved but have their contents zeroed.
 * Experimentation has shown that the device hangs onto old async event queue
 * addresses, even through a reset, with surprising results if the addresses
 * happen to change.
 *
 * We clean up all of the Tx and Rx ring descriptors and the TCBs but leave the
 * allocated memory for the ring data and mutexes intact. Pointers to this
 * memory have already been provided to MAC, and the mutexes keep the rings
 * locked until we're ready to start them again.
 *
 * To ensure that other driver activity is excluded, we hold the mutexes on the
 * Tx and Rx rings throughout, and unset the `ENA_STATE_STARTED` bit in the
 * state, which causes the interrupt handlers to return without doing any work.
 * The admin interrupt, used for notifications of admin completions or new
 * asynchronous events, is masked after the device is reset until we're ready
 * to process them again.
 */
bool
ena_reset(ena_t *ena, const enahw_reset_reason_t reason)
{
        ena_txq_state_t tx_state[ena->ena_num_txqs];
        ena_rxq_state_t rx_state[ena->ena_num_rxqs];
        bool ret = false;

        ena_err(ena, "resetting device with reason 0x%x [%s]",
            reason, enahw_reset_reason(reason));

        VERIFY0(ena->ena_state & ENA_STATE_RESETTING);
        atomic_or_32(&ena->ena_state, ENA_STATE_RESETTING);

        VERIFY(ena->ena_state & ENA_STATE_STARTED);
        atomic_and_32(&ena->ena_state, ~ENA_STATE_STARTED);

        mutex_enter(&ena->ena_lock);

        ena_update_regcache(ena);

        for (uint_t i = 0; i < ena->ena_num_txqs; i++) {
                ena_txq_t *txq = &ena->ena_txqs[i];

                mutex_enter(&txq->et_lock);
                tx_state[i] = txq->et_state;
                if (txq->et_state & ENA_TXQ_STATE_RUNNING)
                        ena_ring_tx_stop((mac_ring_driver_t)txq);
        }

        for (uint_t i = 0; i < ena->ena_num_rxqs; i++) {
                ena_rxq_t *rxq = &ena->ena_rxqs[i];

                mutex_enter(&rxq->er_lock);
                rx_state[i] = rxq->er_state;
                if (rxq->er_state & ENA_RXQ_STATE_RUNNING)
                        ena_ring_rx_stop((mac_ring_driver_t)rxq);
        }

        if (!ena_device_reset(ena, reason)) {
                ena_err(ena, "reset: failed to reset device");
                goto out;
        }

        /* This masks the admin/aenq interrupt */
        ena_hw_bar_write32(ena, ENAHW_REG_INTERRUPT_MASK, ENAHW_INTR_MASK);

        ena_cleanup_txqs(ena, true);
        ena_cleanup_rxqs(ena, true);

        ena_release_all_cmd_ctx(ena);

        if (!ena_admin_cq_init(ena) || !ena_admin_sq_init(ena)) {
                ena_err(ena, "reset: failed to program admin queues");
                goto out;
        }

        if (!ena_init_host_info(ena)) {
                ena_err(ena, "reset: failed to set host info");
                goto out;
        }

        if (!ena_aenq_init(ena) || !ena_aenq_configure(ena)) {
                ena_err(ena, "reset: failed to configure aenq");
                goto out;
        }

        if (!ena_set_mtu(ena)) {
                ena_err(ena, "reset: failed to set MTU");
                goto out;
        }

        if (!ena_attach_alloc_txqs(ena) || !ena_attach_alloc_rxqs(ena)) {
                ena_err(ena, "reset: failed to program IO queues");
                goto out;
        }

        ena_aenq_enable(ena);
        ena_hw_bar_write32(ena, ENAHW_REG_INTERRUPT_MASK, ENAHW_INTR_UNMASK);

        for (uint_t i = 0; i < ena->ena_num_rxqs; i++) {
                ena_rxq_t *rxq = &ena->ena_rxqs[i];

                mutex_exit(&rxq->er_lock);
                if (rx_state[i] & ENA_RXQ_STATE_RUNNING) {
                        (void) ena_ring_rx_start((mac_ring_driver_t)rxq,
                            rxq->er_m_gen_num);
                }
        }

        for (uint_t i = 0; i < ena->ena_num_txqs; i++) {
                ena_txq_t *txq = &ena->ena_txqs[i];

                mutex_exit(&txq->et_lock);
                if (tx_state[i] & ENA_TXQ_STATE_RUNNING) {
                        (void) ena_ring_tx_start((mac_ring_driver_t)txq,
                            txq->et_m_gen_num);
                }
        }

        atomic_or_32(&ena->ena_state, ENA_STATE_STARTED);
        ret = true;

out:
        atomic_and_32(&ena->ena_state, ~ENA_STATE_RESETTING);
        mutex_exit(&ena->ena_lock);

        ena_update_regcache(ena);

        return (ret);
}

ena_attach_desc_t ena_attach_tbl[ENA_ATTACH_NUM_ENTRIES] = {
        {
                .ead_seq = ENA_ATTACH_PCI,
                .ead_name = "PCI config",
                .ead_attach_fn = ena_attach_pci,
                .ead_attach_hard_fail = true,
                .ead_cleanup_fn = ena_cleanup_pci,
        },

        {
                .ead_seq = ENA_ATTACH_REGS,
                .ead_name = "BAR mapping",
                .ead_attach_fn = ena_attach_regs_map,
                .ead_attach_hard_fail = true,
                .ead_cleanup_fn = ena_cleanup_regs_map,
        },

        {
                .ead_seq = ENA_ATTACH_DEV_INIT,
                .ead_name = "device initialization",
                .ead_attach_fn = ena_attach_device_init,
                .ead_attach_hard_fail = true,
                .ead_cleanup_fn = ena_cleanup_device_init,
        },

        {
                .ead_seq = ENA_ATTACH_READ_CONF,
                .ead_name = "ena.conf",
                .ead_attach_fn = ena_attach_read_conf,
                .ead_attach_hard_fail = true,
                .ead_cleanup_fn = NULL,
        },

        {
                .ead_seq = ENA_ATTACH_DEV_CFG,
                .ead_name = "device config",
                .ead_attach_fn = ena_attach_dev_cfg,
                .ead_attach_hard_fail = true,
                .ead_cleanup_fn = NULL,
        },

        {
                .ead_seq = ENA_ATTACH_INTR_ALLOC,
                .ead_name = "interrupt allocation",
                .ead_attach_fn = ena_attach_intr_alloc,
                .ead_attach_hard_fail = true,
                .ead_cleanup_fn = ena_cleanup_intr_alloc,
        },

        {
                .ead_seq = ENA_ATTACH_INTR_HDLRS,
                .ead_name = "interrupt handlers",
                .ead_attach_fn = ena_intr_add_handlers,
                .ead_attach_hard_fail = true,
                .ead_cleanup_fn = ena_intr_remove_handlers,
        },

        {
                .ead_seq = ENA_ATTACH_TXQS_ALLOC,
                .ead_name = "Tx queues",
                .ead_attach_fn = ena_attach_alloc_txqs,
                .ead_attach_hard_fail = true,
                .ead_cleanup_fn = ena_cleanup_txqs,
        },

        {
                .ead_seq = ENA_ATTACH_RXQS_ALLOC,
                .ead_name = "Rx queues",
                .ead_attach_fn = ena_attach_alloc_rxqs,
                .ead_attach_hard_fail = true,
                .ead_cleanup_fn = ena_cleanup_rxqs,
        },

        /*
         * The chance of mac_unregister() failure poses a problem to
         * cleanup. We address interrupt disablement and mac
         * unregistration explicitly in the attach/detach routines.
         */
        {
                .ead_seq = ENA_ATTACH_MAC_REGISTER,
                .ead_name = "mac registration",
                .ead_attach_fn = ena_mac_register,
                .ead_attach_hard_fail = true,
                .ead_cleanup_fn = NULL,
        },

        {
                .ead_seq = ENA_ATTACH_INTRS_ENABLE,
                .ead_name = "enable interrupts",
                .ead_attach_fn = ena_intrs_enable,
                .ead_attach_hard_fail = true,
                .ead_cleanup_fn = NULL,
        }
};

/*
 * This function undoes any work done by ena_attach(), either in
 * response to a failed attach or a planned detach. At the end of this
 * function ena_attach_seq should be zero, otherwise it means
 * something has not be freed/uninitialized.
 */
static void
ena_cleanup(ena_t *ena)
{
        if (ena == NULL || ena->ena_attach_seq == 0) {
                return;
        }

        /*
         * We VERIFY this because if the seq is greater than entries
         * we drift into space and execute god knows what.
         */
        VERIFY3U(ena->ena_attach_seq, <, ENA_ATTACH_NUM_ENTRIES);

        while (ena->ena_attach_seq > 0) {
                int idx = ena->ena_attach_seq - 1;
                ena_attach_desc_t *desc = &ena_attach_tbl[idx];

                ena_dbg(ena, "running cleanup sequence: %s (%d)",
                    desc->ead_name, idx);

                if (desc->ead_cleanup_fn != NULL)
                        desc->ead_cleanup_fn(ena, false);
                ena->ena_attach_seq--;
        }

        ASSERT3U(ena->ena_attach_seq, ==, 0);
        mutex_destroy(&ena->ena_lock);
        mutex_destroy(&ena->ena_watchdog_lock);
}

static int
ena_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
        ena_t *ena;

        if (cmd != DDI_ATTACH) {
                return (DDI_FAILURE);
        }

        ena = kmem_zalloc(sizeof (ena_t), KM_SLEEP);
        ena->ena_instance = ddi_get_instance(dip);
        ena->ena_dip = dip;
        ena->ena_instance = ddi_get_instance(dip);
        ena->ena_page_sz = ddi_ptob(dip, 1);

        for (int i = 0; i < ENA_ATTACH_NUM_ENTRIES; i++) {
                bool success;
                ena_attach_desc_t *desc = &ena_attach_tbl[i];

                ena_dbg(ena, "running attach sequence: %s (%d)", desc->ead_name,
                    i);

                if (!(success = desc->ead_attach_fn(ena))) {
                        ena_err(ena, "attach sequence failed: %s (%d)",
                            desc->ead_name, i);

                        if (ena->ena_attach_seq == ENA_ATTACH_MAC_REGISTER) {
                                /*
                                 * In this specific case
                                 * ENA_ATTACH_INTRS_ENABLE has failed,
                                 * and we may or may not be able to
                                 * unregister the mac, depending on if
                                 * something in userspace has created
                                 * a client on top.
                                 *
                                 * NOTE: Something that would be nice
                                 * to add to mac is the ability to
                                 * register a provider separate from
                                 * "publishing" it to the rest of the
                                 * system. This would allow a driver
                                 * to register its mac, do some
                                 * additional work that might fail,
                                 * and then unregister if that work
                                 * fails without concern for any
                                 * chance of failure when calling
                                 * unregister. This would remove the
                                 * complexity of the situation we are
                                 * trying to address here, as we would
                                 * know that until the mac has been
                                 * "published", there is no chance for
                                 * mac_unregister() to fail.
                                 */
                                if (ena_mac_unregister(ena) != 0) {
                                        return (DDI_FAILURE);
                                }

                                ena->ena_attach_seq--;
                        } else {
                                /*
                                 * Since the ead_seq is predicated on
                                 * successful ead_attach_fn we must
                                 * run the specific cleanup handler
                                 * before calling the global cleanup
                                 * routine. This also means that all
                                 * cleanup functions must be able to
                                 * deal with partial success of the
                                 * corresponding ead_attach_fn.
                                 */
                                if (desc->ead_cleanup_fn != NULL)
                                        desc->ead_cleanup_fn(ena, false);
                        }

                        ena_cleanup(ena);
                        kmem_free(ena, sizeof (ena_t));
                        return (DDI_FAILURE);
                }

                if (success) {
                        ena_dbg(ena, "attach sequence completed: %s (%d)",
                            desc->ead_name, i);
                }

                ena->ena_attach_seq = desc->ead_seq;
        }

        /*
         * Now that interrupts are enabled, unmask the admin interrupt.
         * Note that this interrupt is generated for both the admin queue and
         * the AENQ, but this driver always polls the admin queue. The surplus
         * interrupt for admin command completion triggers a harmless check of
         * the AENQ.
         */
        ena_hw_bar_write32(ena, ENAHW_REG_INTERRUPT_MASK, ENAHW_INTR_UNMASK);
        ena_aenq_enable(ena);

        ddi_set_driver_private(dip, ena);

        ena_update_regcache(ena);

        atomic_or_32(&ena->ena_state, ENA_STATE_INITIALIZED);

        return (DDI_SUCCESS);
}

static int
ena_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
        ena_t *ena = ddi_get_driver_private(dip);

        if (ena == NULL) {
                return (DDI_FAILURE);
        }

        /*
         * Before we can proceed to cleanup we have to treat
         * mac_unregister() explicitly -- if there are still
         * outstanding clients, then we can't proceed with detach or
         * cleanup.
         */

        /*
         * Why this would fail I don't know, but if we proceed to mac
         * unregister, then there is a good chance we will panic in
         * the Rx interrupt handler when calling mac_rx_ring()
         */
        if (!ena_intrs_disable(ena)) {
                return (DDI_FAILURE);
        }

        /* We can't detach if clients are actively using the device. */
        if (ena_mac_unregister(ena) != 0) {
                (void) ena_intrs_enable(ena);
                return (DDI_FAILURE);
        }

        /*
         * At this point we can proceed with the rest of cleanup on a
         * best-effort basis.
         */
        ena->ena_attach_seq = ENA_ATTACH_RXQS_ALLOC;
        ena_cleanup(ena);
        ddi_set_driver_private(dip, NULL);
        kmem_free(ena, sizeof (ena_t));
        return (DDI_SUCCESS);
}

static struct cb_ops ena_cb_ops = {
        .cb_open = nodev,
        .cb_close = nodev,
        .cb_strategy = nodev,
        .cb_print = nodev,
        .cb_dump = nodev,
        .cb_read = nodev,
        .cb_write = nodev,
        .cb_ioctl = nodev,
        .cb_devmap = nodev,
        .cb_mmap = nodev,
        .cb_segmap = nodev,
        .cb_chpoll = nochpoll,
        .cb_prop_op = ddi_prop_op,
        .cb_flag = D_MP,
        .cb_rev = CB_REV,
        .cb_aread = nodev,
        .cb_awrite = nodev
};

static struct dev_ops ena_dev_ops = {
        .devo_rev = DEVO_REV,
        .devo_refcnt = 0,
        .devo_getinfo = NULL,
        .devo_identify = nulldev,
        .devo_probe = nulldev,
        .devo_attach = ena_attach,
        .devo_detach = ena_detach,
        .devo_reset = nodev,
        .devo_quiesce = ddi_quiesce_not_supported,
        .devo_cb_ops = &ena_cb_ops
};

static struct modldrv ena_modldrv = {
        .drv_modops = &mod_driverops,
        .drv_linkinfo = "AWS ENA Ethernet",
        .drv_dev_ops = &ena_dev_ops
};

static struct modlinkage ena_modlinkage = {
        .ml_rev = MODREV_1,
        .ml_linkage = { &ena_modldrv, NULL }
};

int
_init(void)
{
        int ret;

        mac_init_ops(&ena_dev_ops, ENA_MODULE_NAME);

        if ((ret = mod_install(&ena_modlinkage)) != 0) {
                mac_fini_ops(&ena_dev_ops);
                return (ret);
        }

        return (ret);
}

int
_info(struct modinfo *modinfop)
{
        return (mod_info(&ena_modlinkage, modinfop));
}

int
_fini(void)
{
        int ret;

        if ((ret = mod_remove(&ena_modlinkage)) != 0) {
                return (ret);
        }

        mac_fini_ops(&ena_dev_ops);
        return (ret);
}