/*
 * This file and its contents are supplied under the terms of the
 * Common Development and Distribution License ("CDDL"), version 1.0.
 * You may only use this file in accordance with the terms of version
 * 1.0 of the CDDL.
 *
 * A full copy of the text of the CDDL should have accompanied this
 * source.  A copy of the CDDL is also available via the Internet at
 * http://www.illumos.org/license/CDDL.
 */

/*
 * Copyright 2026 Oxide Computer Company
 */

#ifndef _ENA_H
#define _ENA_H

#include <sys/stdbool.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/types.h>
#include <sys/atomic.h>
#include <sys/list.h>
#include <sys/time.h>
#include <sys/modctl.h>
#include <sys/conf.h>
#include <sys/cpuvar.h>
#include <sys/pci.h>
#include <sys/sysmacros.h>
#include <sys/mac.h>
#include <sys/mac_ether.h>
#include <sys/mac_provider.h>
#include <sys/pattr.h>
#include <sys/strsun.h>
#include <sys/ethernet.h>
#include <sys/vlan.h>
#include <sys/utsname.h>
#include "ena_hw.h"

/*
 * AWS ENA Ethernet Driver
 */

#ifdef __cplusplus
extern "C" {
#endif

#define ENA_MODULE_NAME "ena"

/*
 * The minimum supported ENA device controller version.
 */
#define ENA_CTRL_MAJOR_VSN_MIN          0
#define ENA_CTRL_MINOR_VSN_MIN          0
#define ENA_CTRL_SUBMINOR_VSN_MIN       1

/*
 * These values are used to communicate the driver version to the AWS
 * hypervisor via the HOST_ATTR_CONFIG feature. We don't know what
 * exactly AWS does with this info, but it's fairly safe to assume
 * it's used solely for debug/informational purposes. The Linux driver
 * updates these values frequently as bugs are fixed and features are
 * added. Some instance types have minimum driver version requirements
 * but they do not appear to check the version we pass.
 */
#define ENA_MODULE_VER_MAJOR    1
#define ENA_MODULE_VER_MINOR    1
#define ENA_MODULE_VER_SUBMINOR 0

/*
 * The Linux driver doesn't document what the specification version
 * number controls or the contract around version changes. The best we
 * can do is use the same version that they use and port version
 * changes as they come (the last one was in 2018).
 *
 * common: ENA_COMMON_SPEC_VERSION_{MAJOR,MINOR}
 */
#define ENA_SPEC_VERSION_MAJOR  2
#define ENA_SPEC_VERSION_MINOR  0


/* PCI BAR index for the register BAR */
#define ENA_REG_BAR     0
/* PCI BAR index for the LLQ memory BAR */
#define ENA_LLQ_BAR     2

/*
 * A sentinel value passed as argument to ena_ring_rx() to indicate
 * the Rx ring is being read in interrupt mode, not polling mode.
 */
#define ENA_INTERRUPT_MODE      -1

#define ENA_RX_BUF_IPHDR_ALIGNMENT      2
#define ENA_ADMINQ_DEPTH                32
#define ENA_AENQ_NUM_DESCS              32

/* Convert milliseconds to nanoseconds. */
#define ENA_MS_TO_NS(ms)        ((ms) * 1000000ul)

/*
 * The default amount of time we will wait for an admin command to complete,
 * specified in nanoseconds. This can be overridden by hints received from the
 * device. We default to half a second.
 */
#define ENA_ADMIN_CMD_DEF_TIMEOUT_NS    MSEC2NSEC(500)

/*
 * The interval of the watchdog timer, in nanoseconds.
 */
#define ENA_WATCHDOG_INTERVAL_NS        MSEC2NSEC(1000)

/*
 * The device sends a keepalive message every second. If we don't see any for
 * a while we will trigger a device reset. Other open source drivers use
 * 6 seconds for this value, so do we.
 */
#define ENA_DEVICE_KEEPALIVE_TIMEOUT_NS MSEC2NSEC(6000)

/*
 * The number of consecutive times a TX queue needs to be seen as blocked by
 * the watchdog timer before a reset is invoked. Since the watchdog interval
 * is one second, this is approximately in seconds.
 */
#define ENA_TX_STALL_TIMEOUT            8

/*
 * In order to avoid rapidly sending basic stats requests to the controller, we
 * impose a limit of one request every 10ms.
 */
#define ENA_BASIC_STATS_MINIMUM_INTERVAL_NS     MSEC2NSEC(10);

/*
 * Property macros.
 */
#define ENA_PROP_RXQ_NUM_DESCS  "rx_queue_num_descs"
#define ENA_PROP_RXQ_NUM_DESCS_MIN      64

#define ENA_PROP_TXQ_NUM_DESCS  "tx_queue_num_descs"
#define ENA_PROP_TXQ_NUM_DESCS_MIN      64

#define ENA_PROP_RXQ_INTR_LIMIT "rx_queue_intr_limit"
#define ENA_PROP_RXQ_INTR_LIMIT_MIN     16
#define ENA_PROP_RXQ_INTR_LIMIT_MAX     4096
#define ENA_PROP_RXQ_INTR_LIMIT_DEF     256

#define ENA_DMA_BIT_MASK(x)     ((1ULL << (x)) - 1ULL)
#define ENA_DMA_VERIFY_ADDR(ena, phys_addr)                             \
        VERIFY3U(ENA_DMA_BIT_MASK((ena)->ena_dma_width) & (phys_addr), \
            ==, (phys_addr))

typedef struct ena_dma_conf {
        size_t          edc_size;
        uint64_t        edc_align;
        int             edc_sgl;
        uchar_t         edc_endian;
        bool            edc_stream;
} ena_dma_conf_t;

typedef struct ena_dma_buf {
        caddr_t                 edb_va;
        size_t                  edb_len;
        /*
         * The length given by DMA engine, kept around for debugging
         * purposes.
         */
        size_t                  edb_real_len;
        size_t                  edb_used_len;
        ddi_acc_handle_t        edb_acc_hdl;
        ddi_dma_handle_t        edb_dma_hdl;
        const ddi_dma_cookie_t  *edb_cookie;
} ena_dma_buf_t;

/*
 * We always sync the entire range, and therefore expect success.
 */
#ifdef DEBUG
#define ENA_DMA_SYNC(buf, flag)                                 \
        ASSERT0(ddi_dma_sync((buf).edb_dma_hdl, 0, 0, (flag)))
#else  /* DEBUG */
#define ENA_DMA_SYNC(buf, flag)                                 \
        ((void)ddi_dma_sync((buf).edb_dma_hdl, 0, 0, (flag)))
#endif

typedef void (*ena_aenq_hdlr_t)(void *data, enahw_aenq_desc_t *desc);

typedef struct ena_aenq {
        enahw_aenq_desc_t       *eaenq_descs;
        ena_dma_buf_t           eaenq_dma;
        ena_aenq_hdlr_t         eaenq_hdlrs[ENAHW_AENQ_GROUPS_ARR_NUM];
        uint16_t                eaenq_num_descs;
        uint16_t                eaenq_head;
        uint8_t                 eaenq_phase;
} ena_aenq_t;

typedef struct ena_admin_sq {
        enahw_cmd_desc_t        *eas_entries;
        ena_dma_buf_t           eas_dma;
        uint32_t                *eas_dbaddr;
        uint16_t                eas_tail;
        uint8_t                 eas_phase;
} ena_admin_sq_t;

typedef struct ena_admin_cq {
        enahw_resp_desc_t       *eac_entries;
        ena_dma_buf_t           eac_dma;
        uint16_t                eac_head;
        uint8_t                 eac_phase;
} ena_admin_cq_t;

/*
 * The command context is used to track outstanding requests and match
 * them to device responses.
 */
typedef struct ena_cmd_ctx {
        list_node_t             ectx_node;

        /*
         * The index into ea_cmd_ctxs where this ctx lives. Used as
         * the command ID value in the command descriptor. This allows
         * us to match a response to its associated context.
         */
        uint16_t                ectx_id;

        /* Is the command pending? */
        bool                    ectx_pending;

        /* The type of command associated with this context. */
        enahw_cmd_opcode_t      ectx_cmd_opcode;

        /*
         * The location to copy the full response to. This is
         * specified by the caller of the command during
         * submission.
         */
        enahw_resp_desc_t       *ectx_resp;
} ena_cmd_ctx_t;

/*
 * The admin queue, the queue through which commands are sent to the
 * device.
 *
 * WO: Write Once (at initialization)
 *
 * In general, only a single lock needs to be held in order to access
 * the different parts of the admin queue:
 *
 *  sq_lock: Any data dealing with submitting admin commands, which
 *  includes acquiring a command context.
 *
 *  cq_lock: Any data dealing with reading command responses.
 *
 *  stat_lock: For accessing statistics.
 *
 * In some cases, the ectx_lock/stat_lock may be held in tandem with
 * either the SQ or CQ lock. In that case, the SQ/CQ lock is always
 * entered first.
 */
typedef struct ena_adminq {
        kmutex_t                ea_sq_lock;     /* WO */
        kmutex_t                ea_cq_lock;     /* WO */
        kmutex_t                ea_stat_lock;   /* WO */

        hrtime_t                ea_cmd_timeout_ns; /* WO */

        uint16_t                ea_qlen;        /* WO */
        bool                    ea_poll_mode;   /* WO */

        ena_cmd_ctx_t           *ea_cmd_ctxs;     /* WO */
        list_t                  ea_cmd_ctxs_free; /* ea_sq_lock */
        list_t                  ea_cmd_ctxs_used; /* ea_sq_lock */
        uint16_t                ea_pending_cmds; /* ea_sq_lock */
        ena_admin_sq_t          ea_sq; /* eq_sq_lock */
        ena_admin_cq_t          ea_cq; /* eq_cq_lock */

        /* ea_stat_lock */
        struct ena_adminq_stats {
                uint64_t cmds_fail;
                uint64_t cmds_submitted;
                uint64_t cmds_success;
                uint64_t queue_full;
        } ea_stats;
} ena_adminq_t;

/*
 * Cache of the last set of value hints received from the device. See the
 * definition of ehahw_device_hints_t in ena_hw.h for more detail on the
 * purpose of each.
 */
typedef struct ena_hints {
        uint16_t                eh_mmio_read_timeout;
        uint16_t                eh_keep_alive_timeout;
        uint16_t                eh_tx_comp_timeout;
        uint16_t                eh_missed_tx_reset_threshold;
        uint16_t                eh_admin_comp_timeout;
        uint16_t                eh_max_tx_sgl;
        uint16_t                eh_max_rx_sgl;
} ena_hints_t;

typedef enum ena_attach_seq {
        ENA_ATTACH_PCI = 1,      /* PCI config space */
        ENA_ATTACH_REGS,         /* BAR mapping */
        ENA_ATTACH_DEV_INIT,     /* ENA device initialization */
        ENA_ATTACH_READ_CONF,    /* Read driver conf file */
        ENA_ATTACH_DEV_CFG,      /* Set any needed device config */
        ENA_ATTACH_INTR_ALLOC,   /* interrupt handles allocated */
        ENA_ATTACH_INTR_HDLRS,   /* intr handlers set */
        ENA_ATTACH_TXQS_ALLOC,   /* Tx Queues allocated */
        ENA_ATTACH_RXQS_ALLOC,   /* Tx Queues allocated */
        ENA_ATTACH_MAC_REGISTER, /* registered with mac */
        ENA_ATTACH_INTRS_ENABLE, /* interrupts are enabled */
        ENA_ATTACH_END
} ena_attach_seq_t;

#define ENA_ATTACH_SEQ_FIRST    (ENA_ATTACH_PCI)
#define ENA_ATTACH_NUM_ENTRIES  (ENA_ATTACH_END - 1)

struct ena;
typedef bool (*ena_attach_fn_t)(struct ena *);
typedef void (*ena_cleanup_fn_t)(struct ena *, bool);

typedef struct ena_attach_desc {
        ena_attach_seq_t ead_seq;
        const char *ead_name;
        ena_attach_fn_t ead_attach_fn;
        bool ead_attach_hard_fail;
        ena_cleanup_fn_t ead_cleanup_fn;
} ena_attach_desc_t;

typedef enum {
        ENA_TCB_NONE,
        ENA_TCB_COPY
} ena_tcb_type_t;

/*
 * The TCB is used to track information relating to the Tx of a
 * packet. At the moment we support copy only.
 */
typedef struct ena_tx_control_block {
        /*
         * The index into et_tcbs where this tcb lives. Used as the request ID
         * value in the Tx descriptor. This allows us to match a response to
         * its associated TCB.
         */
        uint16_t        etcb_id;
        mblk_t          *etcb_mp;
        ena_tcb_type_t  etcb_type;
        ena_dma_buf_t   etcb_dma;
} ena_tx_control_block_t;

typedef enum ena_txq_state {
        ENA_TXQ_STATE_NONE              = 0,
        ENA_TXQ_STATE_HOST_ALLOC        = 1 << 0,
        ENA_TXQ_STATE_CQ_CREATED        = 1 << 1,
        ENA_TXQ_STATE_SQ_CREATED        = 1 << 2,
        ENA_TXQ_STATE_READY             = 1 << 3, /* TxQ ready and waiting */
        ENA_TXQ_STATE_RUNNING           = 1 << 4, /* intrs enabled */
} ena_txq_state_t;

typedef struct ena_txq_stat {
        /* Number of times mac_ether_offload_info() has failed. */
        kstat_named_t   ets_hck_meoifail;

        /*
         * Total number of times the ring was blocked due to
         * insufficient descriptors, or unblocked due to recycling
         * descriptors.
         */
        kstat_named_t   ets_blocked;
        kstat_named_t   ets_unblocked;

        /* The total number descriptors that have been recycled. */
        kstat_named_t   ets_recycled;

        /*
         * Number of bytes and packets that have been _submitted_ to
         * the device.
         */
        kstat_named_t   ets_bytes;
        kstat_named_t   ets_packets;
} ena_txq_stat_t;

/*
 * A transmit queue, made up of a Submission Queue (SQ) and Completion
 * Queue (CQ) to form a logical descriptor ring for sending packets.
 *
 * Write Once (WO)
 *
 *   This value is written once, before the datapath is activated, in
 *   a function which is controlled by mac(9E). Some values may be
 *   written earlier, during ena attach, like et_ena and
 *   et_sq_num_descs.
 *
 * Tx Mutex (TM) -- et_lock
 *
 *   This value is protected by the Tx queue's mutex. Some values may
 *   be initialized in a WO path, but also continually updated as part
 *   of normal datapath operation, such as et_sq_avail_descs. These
 *   values need mutex protection.
 */
typedef struct ena_txq {
        kmutex_t                et_lock; /* WO */

        struct ena              *et_ena; /* WO */
        uint_t                  et_txqs_idx; /* WO */
        mac_ring_handle_t       et_mrh;  /* WO */
        uint64_t                et_m_gen_num; /* TM */
        ena_txq_state_t         et_state; /* WO */
        uint16_t                et_intr_vector; /* WO */

        enahw_tx_desc_t         *et_sq_descs; /* TM */
        ena_dma_buf_t           et_sq_dma;    /* WO */

        /* Is the Tx queue currently in a blocked state? */
        bool                    et_blocked; /* TM */

        /*
         * The number of descriptors owned by this ring. This value
         * never changes after initialization.
         */
        uint16_t                et_sq_num_descs;   /* WO */

        /*
         * The number of descriptors currently available for Tx
         * submission. When this value reaches zero the ring must
         * block until device notifies us of freed descriptors.
         */
        uint16_t                et_sq_avail_descs; /* TM */

        /*
         * The current tail index of the queue (the first free
         * descriptor for host Tx submission). After initialization,
         * this value only increments, relying on unsigned wrap
         * around. The ENA device seems to expect this behavior,
         * performing its own modulo on the value for the purposes of
         * indexing, much like the driver code needs to do in order to
         * access the proper TCB entry.
         */
        uint16_t                et_sq_tail_idx;  /* TM */

        /*
         * The phase is used to know which CQ descriptors may be
         * reclaimed. This is explained further in ena.c.
         */
        uint16_t                et_sq_phase; /* TM */
        uint16_t                et_sq_hw_idx; /* WO */

        /*
         * The "doorbell" address is how the host indicates to the
         * device which descriptors are ready for Tx processing.
         */
        uint32_t                *et_sq_db_addr; /* WO */

        /*
         * LLQ state for this queue. When LLQ is active descriptors
         * are written to device memory at et_sq_llq_addr rather than
         * to host memory at et_sq_descs.
         *
         * The bounce buffer (et_sq_llq_buf) is used to stage a
         * complete LLQ entry before copying it to device memory.
         *
         * The burst counter (et_sq_llq_entries_left) tracks the
         * number of LLQ entries that may be written before the next
         * doorbell, for the LIMIT_TX_BURST acceleration feature.
         */
        void                    *et_sq_llq_addr; /* WO */
        uint8_t                 *et_sq_llq_buf; /* TM */
        uint32_t                et_sq_llq_entries_left; /* TM */

        /*
         * The TCBs track host Tx information, like a pointer to the
         * mblk being submitted. The TCBs currently available for use are
         * maintained in a free list.
         */
        ena_tx_control_block_t  *et_tcbs;    /* TM */
        ena_tx_control_block_t  **et_tcbs_freelist; /* TM */
        uint16_t                et_tcbs_freelist_size; /* TM */

        enahw_tx_cdesc_t        *et_cq_descs; /* TM */
        ena_dma_buf_t           et_cq_dma;    /* WO */
        uint16_t                et_cq_num_descs; /* WO */
        uint16_t                et_cq_head_idx; /* TM */
        uint16_t                et_cq_phase;    /* TM */
        uint16_t                et_cq_hw_idx;   /* WO */

        /*
         * This address is used to control the CQ interrupts.
         */
        uint32_t                *et_cq_unmask_addr; /* WO */
        uint32_t                *et_cq_numa_addr;   /* WO (currently unused) */

        /*
         * This is used to detect transmit stalls and invoke a reset. The
         * watchdog increments this counter when it sees that the TX
         * ring is still blocked, and if it exceeds the threshold then the
         * device is assumed to have stalled and needs to be reset.
         */
        uint32_t                et_stall_watchdog; /* TM */

        /*
         * This mutex protects the Tx queue stats. This mutex may be
         * entered while et_lock is held, but et_lock is not required
         * to access/modify the stats. However, if both locks are
         * held, then et_lock must be entered first.
         */
        kmutex_t                et_stat_lock;
        ena_txq_stat_t          et_stat;
        kstat_t                 *et_kstat;
} ena_txq_t;

typedef enum ena_rxq_state {
        ENA_RXQ_STATE_NONE              = 0,
        ENA_RXQ_STATE_HOST_ALLOC        = 1 << 0,
        ENA_RXQ_STATE_CQ_CREATED        = 1 << 1,
        ENA_RXQ_STATE_SQ_CREATED        = 1 << 2,
        ENA_RXQ_STATE_SQ_FILLED         = 1 << 3,
        ENA_RXQ_STATE_READY             = 1 << 4, /* RxQ ready and waiting */
        ENA_RXQ_STATE_RUNNING           = 1 << 5, /* intrs enabled */
} ena_rxq_state_t;

typedef struct ena_rx_ctrl_block {
        ena_dma_buf_t   ercb_dma;
        uint8_t         ercb_offset;
        uint16_t        ercb_length;
} ena_rx_ctrl_block_t;

typedef enum {
        ENA_RXQ_MODE_POLLING    = 1,
        ENA_RXQ_MODE_INTR       = 2,
} ena_rxq_mode_t;

typedef struct ena_rxq_stat_t {
        /* The total number of packets/bytes received on this queue. */
        kstat_named_t   ers_packets;
        kstat_named_t   ers_bytes;

        /*
         * At this time we expect all incoming frames to fit in a
         * single buffer/descriptor. In some rare event that the
         * device doesn't cooperate this stat is incremented.
         */
        kstat_named_t   ers_multi_desc;

        /*
         * The total number of times we failed to allocate a new mblk
         * for an incoming frame.
         */
        kstat_named_t   ers_allocb_fail;

        /*
         * The total number of times the Rx interrupt handler reached
         * its maximum limit for number of packets to process in a
         * single interrupt. If you see this number increase
         * continuously at a steady rate, then it may be an indication
         * the driver is not entering polling mode.
         */
        kstat_named_t   ers_intr_limit;

        /*
         * The total number of times the device detected an incorrect
         * IPv4 header checksum.
         */
        kstat_named_t   ers_hck_ipv4_err;

        /*
         * The total number of times the device detected an incorrect
         * L4/ULP checksum.
         */
        kstat_named_t   ers_hck_l4_err;
} ena_rxq_stat_t;

/*
 * A receive queue, made up of a Submission Queue (SQ) and Completion
 * Queue (CQ) to form a logical descriptor ring for receiving packets.
 *
 * Write Once (WO)
 *
 *   This value is written once, before the datapath is activated, in
 *   a function which is controlled by mac(9E).
 *
 * Rx Mutex (RM) -- er_lock
 *
 *   This value is protected by the Rx queue's mutex. Some values may
 *   be initialized in a WO path, but also continually updated as part
 *   of normal datapath operation, such as er_sq_avail_descs. These
 *   values need mutex protection.
 */
typedef struct ena_rxq {
        kmutex_t                er_lock;

        struct ena              *er_ena; /* WO */
        uint_t                  er_rxqs_idx; /* WO */
        mac_ring_handle_t       er_mrh;  /* WO */
        uint64_t                er_m_gen_num; /* WO */
        ena_rxq_state_t         er_state; /* WO */
        uint16_t                er_intr_vector; /* WO */
        ena_rxq_mode_t          er_mode;        /* RM */
        uint16_t                er_intr_limit;  /* RM */

        enahw_rx_desc_t         *er_sq_descs; /* RM */
        ena_dma_buf_t           er_sq_dma;    /* WO */
        uint16_t                er_sq_num_descs;   /* WO */
        uint16_t                er_sq_avail_descs; /* RM */
        uint16_t                er_sq_tail_idx;  /* RM */
        uint16_t                er_sq_phase; /* RM */
        uint16_t                er_sq_hw_idx;   /* WO */
        uint32_t                *er_sq_db_addr; /* WO */

        enahw_rx_cdesc_t        *er_cq_descs; /* RM */
        ena_dma_buf_t           er_cq_dma;    /* WO */
        uint16_t                er_cq_num_descs; /* WO */
        uint16_t                er_cq_head_idx;  /* RM */
        uint16_t                er_cq_phase;     /* RM */
        uint16_t                er_cq_hw_idx;    /* WO */
        uint32_t                *er_cq_unmask_addr; /* WO */
        uint32_t                *er_cq_numa_addr;    /* WO (currently unused) */

        ena_rx_ctrl_block_t     *er_rcbs; /* RM */

        kmutex_t                er_stat_lock;
        ena_rxq_stat_t          er_stat;
        kstat_t                 *er_kstat;
} ena_rxq_t;

typedef struct ena_device_stat {
        kstat_named_t   eds_reset_forced;
        kstat_named_t   eds_reset_error;
        kstat_named_t   eds_reset_fatal;
        kstat_named_t   eds_reset_keepalive;
        kstat_named_t   eds_reset_txstall;
} ena_device_stat_t;

/*
 * These are stats based on enahw_resp_basic_stats_t and data that accompanies
 * the asynchronous keepalive event.
 */
typedef struct ena_basic_stat {
        kstat_named_t   ebs_tx_bytes;
        kstat_named_t   ebs_tx_pkts;
        kstat_named_t   ebs_tx_drops;

        kstat_named_t   ebs_rx_bytes;
        kstat_named_t   ebs_rx_pkts;
        kstat_named_t   ebs_rx_drops;
        kstat_named_t   ebs_rx_overruns;
} ena_basic_stat_t;

/* These are stats based on enahw_resp_eni_stats_t. */
typedef struct ena_extended_stat {
        kstat_named_t   ees_bw_in_exceeded;
        kstat_named_t   ees_bw_out_exceeded;
        kstat_named_t   ees_pps_exceeded;
        kstat_named_t   ees_conns_exceeded;
        kstat_named_t   ees_linklocal_exceeded;
} ena_extended_stat_t;

/* These stats monitor which AENQ handlers have been called. */
typedef struct ena_aenq_stat {
        kstat_named_t   eaes_default;
        kstat_named_t   eaes_link_change;
        kstat_named_t   eaes_notification;
        kstat_named_t   eaes_keep_alive;
        kstat_named_t   eaes_request_reset;
        kstat_named_t   eaes_fatal_error;
        kstat_named_t   eaes_warning;
} ena_aenq_stat_t;

#ifdef DEBUG
typedef struct ena_reg {
        const char      *er_name;
        const uint16_t  er_offset;
        uint32_t        er_value;
} ena_reg_t;
#endif

#define ENA_STATE_UNKNOWN       0x00u
#define ENA_STATE_INITIALIZED   0x01u
#define ENA_STATE_STARTED       0x02u
#define ENA_STATE_ERROR         0x04u
#define ENA_STATE_RESETTING     0x08u

/*
 * This structure contains the per-instance (PF of VF) state of the
 * device.
 */
typedef struct ena {
        dev_info_t              *ena_dip;
        int                     ena_instance;

#ifdef DEBUG
        /*
         * In debug kernels, the registers are cached here at various points
         * for easy inspection via mdb(1).
         */
        ena_reg_t               ena_reg[ENAHW_NUM_REGS];
#endif

        /*
         * Global lock, used to synchronize administration changes to
         * the ena_t. This lock should not be held in the datapath.
         */
        kmutex_t                ena_lock;
        ena_attach_seq_t        ena_attach_seq;

        /*
         * We use atomic ops for ena_state so that datapath consumers
         * do not need to enter ena_lock.
         */
        uint32_t                ena_state;

        /*
         * The reason for the last device reset.
         */
        enahw_reset_reason_t    ena_reset_reason;

        /*
         * Watchdog
         */
        kmutex_t                ena_watchdog_lock;
        ddi_periodic_t          ena_watchdog_periodic;
        uint64_t                ena_watchdog_last_keepalive;

        /*
         * PCI config space handle
         */
        ddi_acc_handle_t        ena_pci_hdl;

        /*
         * Memory BAR for registers
         */
        off_t                   ena_reg_size;
        caddr_t                 ena_reg_base;
        ddi_acc_handle_t        ena_reg_hdl;

        /*
         * Memory BAR for LLQ device memory
         */
        bool                    ena_llq_bar_mapped;
        off_t                   ena_llq_bar_size;
        caddr_t                 ena_llq_bar_base;
        ddi_acc_handle_t        ena_llq_bar_hdl;

        /*
         * Vendor information.
         */
        uint16_t                ena_pci_vid;
        uint16_t                ena_pci_did;
        uint8_t                 ena_pci_rev;
        uint16_t                ena_pci_svid;
        uint16_t                ena_pci_sdid;

        /*
         * Device and controller versions.
         */
        uint32_t                ena_dev_major_vsn;
        uint32_t                ena_dev_minor_vsn;
        uint32_t                ena_ctrl_major_vsn;
        uint32_t                ena_ctrl_minor_vsn;
        uint32_t                ena_ctrl_subminor_vsn;
        uint32_t                ena_ctrl_impl_id;

        /*
         * Interrupts
         */
        int                     ena_num_intrs;
        ddi_intr_handle_t       *ena_intr_handles;
        size_t                  ena_intr_handles_sz;
        int                     ena_intr_caps;
        uint_t                  ena_intr_pri;

        mac_handle_t            ena_mh;

        size_t                  ena_page_sz;

        /*
         * The MTU and data layer frame sizes.
         */
        uint32_t                ena_mtu;
        uint32_t                ena_max_frame_hdr;
        uint32_t                ena_max_frame_total;

        /* The size (in bytes) of the Rx/Tx data buffers. */
        uint32_t                ena_tx_buf_sz;
        uint32_t                ena_rx_buf_sz;

        /*
         * The maximum number of Scatter Gather List segments the
         * device can address.
         */
        uint8_t                 ena_tx_sgl_max_sz;
        uint8_t                 ena_rx_sgl_max_sz;

        /* The number of descriptors per Rx/Tx queue. */
        uint16_t                ena_rxq_num_descs;
        uint16_t                ena_txq_num_descs;

        /*
         * The maximum number of frames which may be read per Rx
         * interrupt.
         */
        uint16_t                ena_rxq_intr_limit;

        /* The Rx/Tx data queues (rings). */
        ena_rxq_t               *ena_rxqs;
        uint16_t                ena_num_rxqs;
        ena_txq_t               *ena_txqs;
        uint16_t                ena_num_txqs;

        /* These statistics are device-wide. */
        kstat_t                 *ena_device_kstat;
        ena_device_stat_t       ena_device_stat;
        hrtime_t                ena_device_basic_stat_last_update;
        kmutex_t                ena_device_basic_stat_lock;
        kstat_t                 *ena_device_basic_kstat;
        kstat_t                 *ena_device_extended_kstat;

        /*
         * This tracks AENQ-related stats, it is implicitly
         * device-wide.
         */
        ena_aenq_stat_t         ena_aenq_stat;
        kstat_t                 *ena_aenq_kstat;

        /*
         * The Admin Queue, through which call device commands are
         * sent.
         */
        ena_adminq_t            ena_aq;

        ena_aenq_t              ena_aenq;
        ena_dma_buf_t           ena_host_info;

        /*
         * Hardware info
         */
        ena_hints_t             ena_device_hints;
        uint32_t                ena_supported_features;
        uint32_t                ena_capabilities;
        uint8_t                 ena_dma_width;
        bool                    ena_link_autoneg;
        link_duplex_t           ena_link_duplex;
        uint64_t                ena_link_speed_mbits;
        enahw_link_speeds_t     ena_link_speeds;
        link_state_t            ena_link_state;
        uint32_t                ena_aenq_supported_groups;
        uint32_t                ena_aenq_enabled_groups;

        /*
         * LLQ (Low Latency Queue) state. When ena_llq_enabled is true,
         * TX descriptors are written to device, rather than host, memory.
         */
        bool                                    ena_llq_enabled;
        enahw_llq_header_location_t             ena_llq_header_location;
        enahw_llq_ring_entry_size_t             ena_llq_entry_size;
        uint16_t                                ena_llq_entry_size_bytes;
        enahw_llq_num_descs_before_header_t     ena_llq_num_descs_before_header;
        enahw_llq_stride_ctrl_t                 ena_llq_stride_ctrl;
        uint16_t                                ena_llq_max_tx_burst_size;

        uint32_t                ena_tx_max_sq_num;
        uint32_t                ena_tx_max_sq_num_descs;
        uint32_t                ena_tx_max_cq_num;
        uint32_t                ena_tx_max_cq_num_descs;
        uint16_t                ena_tx_max_desc_per_pkt;
        uint32_t                ena_tx_max_hdr_len;

        uint32_t                ena_rx_max_sq_num;
        uint32_t                ena_rx_max_sq_num_descs;
        uint32_t                ena_rx_max_cq_num;
        uint32_t                ena_rx_max_cq_num_descs;
        uint16_t                ena_rx_max_desc_per_pkt;

        /* This is calculated from the Rx/Tx queue nums. */
        uint16_t                ena_max_io_queues;

        /* Hardware Offloads */
        bool                    ena_tx_l3_ipv4_csum;

        bool                    ena_tx_l4_ipv4_part_csum;
        bool                    ena_tx_l4_ipv4_full_csum;
        bool                    ena_tx_l4_ipv4_lso;

        bool                    ena_tx_l4_ipv6_part_csum;
        bool                    ena_tx_l4_ipv6_full_csum;
        bool                    ena_tx_l4_ipv6_lso;

        bool                    ena_rx_l3_ipv4_csum;
        bool                    ena_rx_l4_ipv4_csum;
        bool                    ena_rx_l4_ipv6_csum;
        bool                    ena_rx_hash;

        uint32_t                ena_max_mtu;
        uint8_t                 ena_mac_addr[ETHERADDRL];
} ena_t;

/*
 * Misc
 */
extern bool ena_reset(ena_t *, const enahw_reset_reason_t);
extern bool ena_is_feat_avail(ena_t *, const enahw_feature_id_t);
extern bool ena_is_cap_avail(ena_t *, const enahw_capability_id_t);
extern void ena_update_hints(ena_t *, enahw_device_hints_t *);

/*
 * Logging functions.
 */
extern bool ena_debug;
extern void ena_err(const ena_t *, const char *, ...) __KPRINTFLIKE(2);
extern void ena_dbg(const ena_t *, const char *, ...) __KPRINTFLIKE(2);
extern void ena_panic(const ena_t *, const char *, ...) __KPRINTFLIKE(2);
extern void ena_trigger_reset(ena_t *, enahw_reset_reason_t);

/*
 * Hardware access.
 */
extern uint32_t ena_hw_bar_read32(const ena_t *, const uint16_t);
extern uint32_t ena_hw_abs_read32(const ena_t *, uint32_t *);
extern void ena_hw_bar_write32(const ena_t *, const uint16_t, const uint32_t);
extern void ena_hw_abs_write32(const ena_t *, uint32_t *, const uint32_t);
extern const char *enahw_reset_reason(enahw_reset_reason_t);
#ifdef DEBUG
extern void ena_init_regcache(ena_t *);
extern void ena_update_regcache(ena_t *);
#else
#define ena_init_regcache(x)
#define ena_update_regcache(x)
#endif

/*
 * Watchdog
 */
extern void ena_enable_watchdog(ena_t *);
extern void ena_disable_watchdog(ena_t *);

/*
 * Stats
 */
extern void ena_stat_device_cleanup(ena_t *);
extern bool ena_stat_device_init(ena_t *);

extern void ena_stat_device_basic_cleanup(ena_t *);
extern bool ena_stat_device_basic_init(ena_t *);

extern void ena_stat_device_extended_cleanup(ena_t *);
extern bool ena_stat_device_extended_init(ena_t *);

extern void ena_stat_aenq_cleanup(ena_t *);
extern bool ena_stat_aenq_init(ena_t *);

extern void ena_stat_rxq_cleanup(ena_rxq_t *);
extern bool ena_stat_rxq_init(ena_rxq_t *);
extern void ena_stat_txq_cleanup(ena_txq_t *);
extern bool ena_stat_txq_init(ena_txq_t *);

/*
 * DMA
 */
extern bool ena_dma_alloc(ena_t *, ena_dma_buf_t *, ena_dma_conf_t *,
    size_t);
extern void ena_dma_free(ena_dma_buf_t *);
extern void ena_dma_bzero(ena_dma_buf_t *);
extern void ena_set_dma_addr(const ena_t *, const uint64_t, enahw_addr_t *);
extern void ena_set_dma_addr_values(const ena_t *, const uint64_t, uint32_t *,
    uint16_t *);

/*
 * Interrupts
 */
extern bool ena_intr_add_handlers(ena_t *);
extern void ena_intr_remove_handlers(ena_t *, bool);
extern void ena_tx_intr_work(ena_txq_t *);
extern void ena_rx_intr_work(ena_rxq_t *);
extern bool ena_intrs_disable(ena_t *);
extern bool ena_intrs_enable(ena_t *);

/*
 * MAC
 */
extern bool ena_mac_register(ena_t *);
extern int ena_mac_unregister(ena_t *);
extern void ena_ring_tx_stop(mac_ring_driver_t);
extern int ena_ring_tx_start(mac_ring_driver_t, uint64_t);
extern mblk_t *ena_ring_tx(void *, mblk_t *);
extern void ena_ring_rx_stop(mac_ring_driver_t);
extern int ena_ring_rx_start(mac_ring_driver_t rh, uint64_t gen_num);
extern int ena_m_stat(void *, uint_t, uint64_t *);
extern mblk_t *ena_ring_rx_poll(void *, int);
extern int ena_ring_rx_stat(mac_ring_driver_t, uint_t, uint64_t *);
extern int ena_ring_tx_stat(mac_ring_driver_t, uint_t, uint64_t *);

/*
 * Admin API
 */
extern int ena_admin_submit_cmd(ena_t *, enahw_cmd_desc_t *,
    enahw_resp_desc_t *, ena_cmd_ctx_t **);
extern int ena_admin_poll_for_resp(ena_t *, ena_cmd_ctx_t *);
extern void ena_free_host_info(ena_t *);
extern bool ena_init_host_info(ena_t *);
extern void ena_create_cmd_ctx(ena_t *);
extern void ena_release_all_cmd_ctx(ena_t *);
extern int ena_create_cq(ena_t *, uint16_t, uint64_t, bool, uint32_t,
    uint16_t *, uint32_t **, uint32_t **);
extern int ena_destroy_cq(ena_t *, uint16_t);
extern int ena_create_sq(ena_t *, uint16_t, uint64_t, bool, uint16_t,
    uint16_t *, uint32_t **, void **);
extern int ena_destroy_sq(ena_t *, uint16_t, bool);
extern int ena_set_feature(ena_t *, enahw_cmd_desc_t *,
    enahw_resp_desc_t *, const enahw_feature_id_t, const uint8_t);
extern int ena_get_feature(ena_t *, enahw_resp_desc_t *,
    const enahw_feature_id_t, const uint8_t);
extern int ena_admin_get_basic_stats(ena_t *, enahw_resp_desc_t *);
extern int ena_admin_get_eni_stats(ena_t *, enahw_resp_desc_t *);
extern int enahw_resp_status_to_errno(ena_t *, enahw_resp_status_t);

/*
 * Async event queue
 */
extern bool ena_aenq_init(ena_t *);
extern bool ena_aenq_configure(ena_t *);
extern void ena_aenq_enable(ena_t *);
extern void ena_aenq_work(ena_t *);
extern void ena_aenq_free(ena_t *);

/*
 * Rx/Tx allocations
 */
extern bool ena_alloc_rxq(ena_rxq_t *);
extern void ena_cleanup_rxq(ena_rxq_t *, bool);
extern bool ena_alloc_txq(ena_txq_t *);
extern void ena_cleanup_txq(ena_txq_t *, bool);

#ifdef __cplusplus
}
#endif

#endif  /* _ENA_H */