/* SPDX-License-Identifier: BSD-3-Clause */
/*  Copyright (c) 2024, Intel Corporation
 *  All rights reserved.
 *
 *  Redistribution and use in source and binary forms, with or without
 *  modification, are permitted provided that the following conditions are met:
 *
 *   1. Redistributions of source code must retain the above copyright notice,
 *      this list of conditions and the following disclaimer.
 *
 *   2. Redistributions in binary form must reproduce the above copyright
 *      notice, this list of conditions and the following disclaimer in the
 *      documentation and/or other materials provided with the distribution.
 *
 *   3. Neither the name of the Intel Corporation nor the names of its
 *      contributors may be used to endorse or promote products derived from
 *      this software without specific prior written permission.
 *
 *  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
 *  AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
 *  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
 *  ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
 *  LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
 *  CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
 *  SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
 *  INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
 *  CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
 *  ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
 *  POSSIBILITY OF SUCH DAMAGE.
 */

/**
 * @file ice_lib.c
 * @brief Generic device setup and sysctl functions
 *
 * Library of generic device functions not specific to the networking stack.
 *
 * This includes hardware initialization functions, as well as handlers for
 * many of the device sysctls used to probe driver status or tune specific
 * behaviors.
 */

#include "ice_lib.h"
#include "ice_iflib.h"
#ifdef PCI_IOV
#include "ice_iov.h"
#endif
#include <dev/pci/pcivar.h>
#include <dev/pci/pcireg.h>
#include <machine/resource.h>
#include <net/if_dl.h>
#include <sys/firmware.h>
#include <sys/priv.h>
#include <sys/limits.h>

/**
 * @var M_ICE
 * @brief main ice driver allocation type
 *
 * malloc(9) allocation type used by the majority of memory allocations in the
 * ice driver.
 */
MALLOC_DEFINE(M_ICE, "ice", "Intel(R) 100Gb Network Driver lib allocations");

/*
 * Helper function prototypes
 */
static int ice_get_next_vsi(struct ice_vsi **all_vsi, int size);
static void ice_set_default_vsi_ctx(struct ice_vsi_ctx *ctx);
static void ice_set_rss_vsi_ctx(struct ice_vsi_ctx *ctx, enum ice_vsi_type type);
static int ice_setup_vsi_qmap(struct ice_vsi *vsi, struct ice_vsi_ctx *ctx);
static int ice_setup_tx_ctx(struct ice_tx_queue *txq,
                            struct ice_tlan_ctx *tlan_ctx, u16 pf_q);
static int ice_setup_rx_ctx(struct ice_rx_queue *rxq);
static int ice_is_rxq_ready(struct ice_hw *hw, int pf_q, u32 *reg);
static void ice_free_fltr_list(struct ice_list_head *list);
static int ice_add_mac_to_list(struct ice_vsi *vsi, struct ice_list_head *list,
                               const u8 *addr, enum ice_sw_fwd_act_type action);
static void ice_check_ctrlq_errors(struct ice_softc *sc, const char *qname,
                                   struct ice_ctl_q_info *cq);
static void ice_process_link_event(struct ice_softc *sc, struct ice_rq_event_info *e);
static void ice_process_ctrlq_event(struct ice_softc *sc, const char *qname,
                                    struct ice_rq_event_info *event);
static void ice_nvm_version_str(struct ice_hw *hw, struct sbuf *buf);
static void ice_update_port_oversize(struct ice_softc *sc, u64 rx_errors);
static void ice_active_pkg_version_str(struct ice_hw *hw, struct sbuf *buf);
static void ice_os_pkg_version_str(struct ice_hw *hw, struct sbuf *buf);
static bool ice_filter_is_mcast(struct ice_vsi *vsi, struct ice_fltr_info *info);
static u_int ice_sync_one_mcast_filter(void *p, struct sockaddr_dl *sdl, u_int errors);
static void ice_add_debug_tunables(struct ice_softc *sc);
static void ice_add_debug_sysctls(struct ice_softc *sc);
static void ice_vsi_set_rss_params(struct ice_vsi *vsi);
static int  ice_set_rss_key(struct ice_vsi *vsi);
static int  ice_set_rss_lut(struct ice_vsi *vsi);
static void ice_set_rss_flow_flds(struct ice_vsi *vsi);
static void ice_clean_vsi_rss_cfg(struct ice_vsi *vsi);
static const char *ice_aq_speed_to_str(struct ice_port_info *pi);
static const char *ice_requested_fec_mode(struct ice_port_info *pi);
static const char *ice_negotiated_fec_mode(struct ice_port_info *pi);
static const char *ice_autoneg_mode(struct ice_port_info *pi);
static const char *ice_flowcontrol_mode(struct ice_port_info *pi);
static void ice_print_bus_link_data(device_t dev, struct ice_hw *hw);
static void ice_set_pci_link_status_data(struct ice_hw *hw, u16 link_status);
static uint8_t ice_pcie_bandwidth_check(struct ice_softc *sc);
static uint64_t ice_pcie_bus_speed_to_rate(enum ice_pcie_bus_speed speed);
static int ice_pcie_lnk_width_to_int(enum ice_pcie_link_width width);
static uint64_t ice_phy_types_to_max_rate(struct ice_port_info *pi);
static void ice_add_sysctls_sw_stats(struct ice_vsi *vsi,
                                     struct sysctl_ctx_list *ctx,
                                     struct sysctl_oid *parent);
static void
ice_add_sysctls_mac_pfc_one_stat(struct sysctl_ctx_list *ctx,
                                 struct sysctl_oid_list *parent_list,
                                 u64* pfc_stat_location,
                                 const char *node_name,
                                 const char *descr);
static void ice_add_sysctls_mac_pfc_stats(struct sysctl_ctx_list *ctx,
                                          struct sysctl_oid *parent,
                                          struct ice_hw_port_stats *stats);
static void ice_setup_vsi_common(struct ice_softc *sc, struct ice_vsi *vsi,
                                 enum ice_vsi_type type, int idx,
                                 bool dynamic);
static void ice_handle_mib_change_event(struct ice_softc *sc,
                                 struct ice_rq_event_info *event);
static void
ice_handle_lan_overflow_event(struct ice_softc *sc,
                              struct ice_rq_event_info *event);
static int ice_add_ethertype_to_list(struct ice_vsi *vsi,
                                     struct ice_list_head *list,
                                     u16 ethertype, u16 direction,
                                     enum ice_sw_fwd_act_type action);
static void ice_del_rx_lldp_filter(struct ice_softc *sc);
static u16 ice_aq_phy_types_to_link_speeds(u64 phy_type_low,
                                           u64 phy_type_high);
struct ice_phy_data;
static int
ice_intersect_phy_types_and_speeds(struct ice_softc *sc,
                                   struct ice_phy_data *phy_data);
static int
ice_apply_saved_phy_req_to_cfg(struct ice_softc *sc,
                               struct ice_aqc_set_phy_cfg_data *cfg);
static int
ice_apply_saved_fec_req_to_cfg(struct ice_softc *sc,
                               struct ice_aqc_set_phy_cfg_data *cfg);
static void
ice_apply_saved_fc_req_to_cfg(struct ice_port_info *pi,
                              struct ice_aqc_set_phy_cfg_data *cfg);
static void
ice_print_ldo_tlv(struct ice_softc *sc,
                  struct ice_link_default_override_tlv *tlv);
static void
ice_sysctl_speeds_to_aq_phy_types(u16 sysctl_speeds, u64 *phy_type_low,
                                  u64 *phy_type_high);
static u16 ice_apply_supported_speed_filter(u16 report_speeds, u8 mod_type);
static void
ice_handle_health_status_event(struct ice_softc *sc,
                               struct ice_rq_event_info *event);
static void
ice_print_health_status_string(device_t dev,
                               struct ice_aqc_health_status_elem *elem);
static void
ice_debug_print_mib_change_event(struct ice_softc *sc,
                                 struct ice_rq_event_info *event);
static bool ice_check_ets_bw(u8 *table);
static u8 ice_dcb_get_num_tc(struct ice_dcbx_cfg *dcbcfg);
static bool
ice_dcb_needs_reconfig(struct ice_softc *sc, struct ice_dcbx_cfg *old_cfg,
                       struct ice_dcbx_cfg *new_cfg);
static void ice_dcb_recfg(struct ice_softc *sc);
static u8 ice_dcb_tc_contig(u8 tc_map);
static int ice_ets_str_to_tbl(const char *str, u8 *table, u8 limit);
static int ice_pf_vsi_cfg_tc(struct ice_softc *sc, u8 tc_map);
static void ice_sbuf_print_ets_cfg(struct sbuf *sbuf, const char *name,
                                   struct ice_dcb_ets_cfg *ets);
static void ice_stop_pf_vsi(struct ice_softc *sc);
static void ice_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt);
static int ice_config_pfc(struct ice_softc *sc, u8 new_mode);
void
ice_add_dscp2tc_map_sysctls(struct ice_softc *sc,
                            struct sysctl_ctx_list *ctx,
                            struct sysctl_oid_list *ctx_list);
static void ice_set_default_local_mib_settings(struct ice_softc *sc);
static bool ice_dscp_is_mapped(struct ice_dcbx_cfg *dcbcfg);
static void ice_start_dcbx_agent(struct ice_softc *sc);
static u16 ice_fw_debug_dump_print_cluster(struct ice_softc *sc,
                                           struct sbuf *sbuf, u16 cluster_id);
static void ice_fw_debug_dump_print_clusters(struct ice_softc *sc,
                                             struct sbuf *sbuf);
static void ice_remove_vsi_mirroring(struct ice_vsi *vsi);
static int ice_get_tx_rx_equalizations(struct ice_hw *hw, u8 serdes_num,
                                       struct ice_serdes_equalization *ptr);
static int ice_fec_counter_read(struct ice_hw *hw, u32 receiver_id,
                                u32 reg_offset, u16 *output);
static int ice_get_port_fec_stats(struct ice_hw *hw, u16 pcs_quad, u16 pcs_port,
                                  struct ice_fec_stats_to_sysctl *fec_stats);
static bool ice_is_serdes_muxed(struct ice_hw *hw);
static int ice_get_maxspeed(struct ice_hw *hw, u8 lport, u8 *max_speed);
static int ice_update_port_topology(u8 lport,
                                    struct ice_port_topology *port_topology,
                                    bool is_muxed);
static int ice_get_port_topology(struct ice_hw *hw, u8 lport,
                                 struct ice_port_topology *port_topology);

static int ice_module_init(void);
static int ice_module_exit(void);

/*
 * package version comparison functions
 */
static bool pkg_ver_empty(struct ice_pkg_ver *pkg_ver, u8 *pkg_name);
static int pkg_ver_compatible(struct ice_pkg_ver *pkg_ver);

/*
 * dynamic sysctl handlers
 */
static int ice_sysctl_show_fw(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_pkg_version(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_os_pkg_version(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_dump_mac_filters(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_dump_vlan_filters(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_dump_ethertype_filters(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_dump_ethertype_mac_filters(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_current_speed(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_request_reset(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_dump_state_flags(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_fec_config(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_fc_config(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_negotiated_fc(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_negotiated_fec(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_phy_type_low(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_phy_type_high(SYSCTL_HANDLER_ARGS);
static int __ice_sysctl_phy_type_handler(SYSCTL_HANDLER_ARGS,
                                         bool is_phy_type_high);
static int ice_sysctl_advertise_speed(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_rx_itr(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_tx_itr(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_fw_lldp_agent(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_fw_cur_lldp_persist_status(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_fw_dflt_lldp_persist_status(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_phy_caps(SYSCTL_HANDLER_ARGS, u8 report_mode);
static int ice_sysctl_phy_sw_caps(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_phy_nvm_caps(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_phy_topo_caps(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_phy_link_status(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_read_i2c_diag_data(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_tx_cso_stat(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_rx_cso_stat(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_pba_number(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_rx_errors_stat(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_dump_dcbx_cfg(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_dump_vsi_cfg(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_dump_phy_stats(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_ets_min_rate(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_up2tc_map(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_pfc_config(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_query_port_ets(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_dscp2tc_map(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_pfc_mode(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_fw_debug_dump_cluster_setting(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_fw_debug_dump_do_dump(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_allow_no_fec_mod_in_auto(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_set_link_active(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_debug_set_link(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_temperature(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_create_mirror_interface(SYSCTL_HANDLER_ARGS);
static int ice_sysctl_destroy_mirror_interface(SYSCTL_HANDLER_ARGS);

/**
 * ice_map_bar - Map PCIe BAR memory
 * @dev: the PCIe device
 * @bar: the BAR info structure
 * @bar_num: PCIe BAR number
 *
 * Maps the specified PCIe BAR. Stores the mapping data in struct
 * ice_bar_info.
 */
int
ice_map_bar(device_t dev, struct ice_bar_info *bar, int bar_num)
{
        if (bar->res != NULL) {
                device_printf(dev, "PCI BAR%d already mapped\n", bar_num);
                return (EDOOFUS);
        }

        bar->rid = PCIR_BAR(bar_num);
        bar->res = bus_alloc_resource_any(dev, SYS_RES_MEMORY, &bar->rid,
                                          RF_ACTIVE);
        if (!bar->res) {
                device_printf(dev, "PCI BAR%d mapping failed\n", bar_num);
                return (ENXIO);
        }

        bar->tag = rman_get_bustag(bar->res);
        bar->handle = rman_get_bushandle(bar->res);
        bar->size = rman_get_size(bar->res);

        return (0);
}

/**
 * ice_free_bar - Free PCIe BAR memory
 * @dev: the PCIe device
 * @bar: the BAR info structure
 *
 * Frees the specified PCIe BAR, releasing its resources.
 */
void
ice_free_bar(device_t dev, struct ice_bar_info *bar)
{
        if (bar->res != NULL)
                bus_release_resource(dev, SYS_RES_MEMORY, bar->rid, bar->res);
        bar->res = NULL;
}

/**
 * ice_set_ctrlq_len - Configure ctrlq lengths for a device
 * @hw: the device hardware structure
 *
 * Configures the control queues for the given device, setting up the
 * specified lengths, prior to initializing hardware.
 */
void
ice_set_ctrlq_len(struct ice_hw *hw)
{
        hw->adminq.num_rq_entries = ICE_AQ_LEN;
        hw->adminq.num_sq_entries = ICE_AQ_LEN;
        hw->adminq.rq_buf_size = ICE_AQ_MAX_BUF_LEN;
        hw->adminq.sq_buf_size = ICE_AQ_MAX_BUF_LEN;

        hw->mailboxq.num_rq_entries = ICE_MBXQ_LEN;
        hw->mailboxq.num_sq_entries = ICE_MBXQ_LEN;
        hw->mailboxq.rq_buf_size = ICE_MBXQ_MAX_BUF_LEN;
        hw->mailboxq.sq_buf_size = ICE_MBXQ_MAX_BUF_LEN;

        hw->sbq.num_rq_entries = ICE_SBQ_LEN;
        hw->sbq.num_sq_entries = ICE_SBQ_LEN;
        hw->sbq.rq_buf_size = ICE_SBQ_MAX_BUF_LEN;
        hw->sbq.sq_buf_size = ICE_SBQ_MAX_BUF_LEN;
}

/**
 * ice_get_next_vsi - Get the next available VSI slot
 * @all_vsi: the VSI list
 * @size: the size of the VSI list
 *
 * Returns the index to the first available VSI slot. Will return size (one
 * past the last index) if there are no slots available.
 */
static int
ice_get_next_vsi(struct ice_vsi **all_vsi, int size)
{
        int i;

        for (i = 0; i < size; i++) {
                if (all_vsi[i] == NULL)
                        return i;
        }

        return size;
}

/**
 * ice_setup_vsi_common - Common VSI setup for both dynamic and static VSIs
 * @sc: the device private softc structure
 * @vsi: the VSI to setup
 * @type: the VSI type of the new VSI
 * @idx: the index in the all_vsi array to use
 * @dynamic: whether this VSI memory was dynamically allocated
 *
 * Perform setup for a VSI that is common to both dynamically allocated VSIs
 * and the static PF VSI which is embedded in the softc structure.
 */
static void
ice_setup_vsi_common(struct ice_softc *sc, struct ice_vsi *vsi,
                     enum ice_vsi_type type, int idx, bool dynamic)
{
        /* Store important values in VSI struct */
        vsi->type = type;
        vsi->sc = sc;
        vsi->idx = idx;
        sc->all_vsi[idx] = vsi;
        vsi->dynamic = dynamic;

        /* Set default mirroring rule information */
        vsi->rule_mir_ingress = ICE_INVAL_MIRROR_RULE_ID;
        vsi->rule_mir_egress = ICE_INVAL_MIRROR_RULE_ID;

        /* Setup the VSI tunables now */
        ice_add_vsi_tunables(vsi, sc->vsi_sysctls);
}

/**
 * ice_alloc_vsi - Allocate a dynamic VSI
 * @sc: device softc structure
 * @type: VSI type
 *
 * Allocates a new dynamic VSI structure and inserts it into the VSI list.
 */
struct ice_vsi *
ice_alloc_vsi(struct ice_softc *sc, enum ice_vsi_type type)
{
        struct ice_vsi *vsi;
        int idx;

        /* Find an open index for a new VSI to be allocated. If the returned
         * index is >= the num_available_vsi then it means no slot is
         * available.
         */
        idx = ice_get_next_vsi(sc->all_vsi, sc->num_available_vsi);
        if (idx >= sc->num_available_vsi) {
                device_printf(sc->dev, "No available VSI slots\n");
                return NULL;
        }

        vsi = (struct ice_vsi *)malloc(sizeof(*vsi), M_ICE, M_NOWAIT | M_ZERO);
        if (!vsi) {
                device_printf(sc->dev, "Unable to allocate VSI memory\n");
                return NULL;
        }

        ice_setup_vsi_common(sc, vsi, type, idx, true);

        return vsi;
}

/**
 * ice_setup_pf_vsi - Setup the PF VSI
 * @sc: the device private softc
 *
 * Setup the PF VSI structure which is embedded as sc->pf_vsi in the device
 * private softc. Unlike other VSIs, the PF VSI memory is allocated as part of
 * the softc memory, instead of being dynamically allocated at creation.
 */
void
ice_setup_pf_vsi(struct ice_softc *sc)
{
        ice_setup_vsi_common(sc, &sc->pf_vsi, ICE_VSI_PF, 0, false);
}

/**
 * ice_alloc_vsi_qmap
 * @vsi: VSI structure
 * @max_tx_queues: Number of transmit queues to identify
 * @max_rx_queues: Number of receive queues to identify
 *
 * Allocates a max_[t|r]x_queues array of words for the VSI where each
 * word contains the index of the queue it represents.  In here, all
 * words are initialized to an index of ICE_INVALID_RES_IDX, indicating
 * all queues for this VSI are not yet assigned an index and thus,
 * not ready for use.
 *
 */
void
ice_alloc_vsi_qmap(struct ice_vsi *vsi, const int max_tx_queues,
                   const int max_rx_queues)
{
        int i;

        MPASS(max_tx_queues > 0);
        MPASS(max_rx_queues > 0);

        /* Allocate Tx queue mapping memory */
        vsi->tx_qmap = malloc(sizeof(u16) * max_tx_queues, M_ICE, M_WAITOK);

        /* Allocate Rx queue mapping memory */
        vsi->rx_qmap = malloc(sizeof(u16) * max_rx_queues, M_ICE, M_WAITOK);

        /* Mark every queue map as invalid to start with */
        for (i = 0; i < max_tx_queues; i++) {
                vsi->tx_qmap[i] = ICE_INVALID_RES_IDX;
        }
        for (i = 0; i < max_rx_queues; i++) {
                vsi->rx_qmap[i] = ICE_INVALID_RES_IDX;
        }
}

/**
 * ice_free_vsi_qmaps - Free the PF qmaps associated with a VSI
 * @vsi: the VSI private structure
 *
 * Frees the PF qmaps associated with the given VSI. Generally this will be
 * called by ice_release_vsi, but may need to be called during attach cleanup,
 * depending on when the qmaps were allocated.
 */
void
ice_free_vsi_qmaps(struct ice_vsi *vsi)
{
        struct ice_softc *sc = vsi->sc;

        if (vsi->tx_qmap) {
                ice_resmgr_release_map(&sc->tx_qmgr, vsi->tx_qmap,
                                           vsi->num_tx_queues);
                free(vsi->tx_qmap, M_ICE);
                vsi->tx_qmap = NULL;
        }

        if (vsi->rx_qmap) {
                ice_resmgr_release_map(&sc->rx_qmgr, vsi->rx_qmap,
                                           vsi->num_rx_queues);
                free(vsi->rx_qmap, M_ICE);
                vsi->rx_qmap = NULL;
        }
}

/**
 * ice_set_default_vsi_ctx - Setup default VSI context parameters
 * @ctx: the VSI context to initialize
 *
 * Initialize and prepare a default VSI context for configuring a new VSI.
 */
static void
ice_set_default_vsi_ctx(struct ice_vsi_ctx *ctx)
{
        u32 table = 0;

        memset(&ctx->info, 0, sizeof(ctx->info));
        /* VSI will be allocated from shared pool */
        ctx->alloc_from_pool = true;
        /* Enable source pruning by default */
        ctx->info.sw_flags = ICE_AQ_VSI_SW_FLAG_SRC_PRUNE;
        /* Traffic from VSI can be sent to LAN */
        ctx->info.sw_flags2 = ICE_AQ_VSI_SW_FLAG_LAN_ENA;
        /* Allow all packets untagged/tagged */
        ctx->info.inner_vlan_flags = ((ICE_AQ_VSI_INNER_VLAN_TX_MODE_ALL &
                                       ICE_AQ_VSI_INNER_VLAN_TX_MODE_M) >>
                                       ICE_AQ_VSI_INNER_VLAN_TX_MODE_S);
        /* Show VLAN/UP from packets in Rx descriptors */
        ctx->info.inner_vlan_flags |= ((ICE_AQ_VSI_INNER_VLAN_EMODE_STR_BOTH &
                                        ICE_AQ_VSI_INNER_VLAN_EMODE_M) >>
                                        ICE_AQ_VSI_INNER_VLAN_EMODE_S);
        /* Have 1:1 UP mapping for both ingress/egress tables */
        table |= ICE_UP_TABLE_TRANSLATE(0, 0);
        table |= ICE_UP_TABLE_TRANSLATE(1, 1);
        table |= ICE_UP_TABLE_TRANSLATE(2, 2);
        table |= ICE_UP_TABLE_TRANSLATE(3, 3);
        table |= ICE_UP_TABLE_TRANSLATE(4, 4);
        table |= ICE_UP_TABLE_TRANSLATE(5, 5);
        table |= ICE_UP_TABLE_TRANSLATE(6, 6);
        table |= ICE_UP_TABLE_TRANSLATE(7, 7);
        ctx->info.ingress_table = CPU_TO_LE32(table);
        ctx->info.egress_table = CPU_TO_LE32(table);
        /* Have 1:1 UP mapping for outer to inner UP table */
        ctx->info.outer_up_table = CPU_TO_LE32(table);
        /* No Outer tag support, so outer_vlan_flags remains zero */
}

/**
 * ice_set_rss_vsi_ctx - Setup VSI context parameters for RSS
 * @ctx: the VSI context to configure
 * @type: the VSI type
 *
 * Configures the VSI context for RSS, based on the VSI type.
 */
static void
ice_set_rss_vsi_ctx(struct ice_vsi_ctx *ctx, enum ice_vsi_type type)
{
        u8 lut_type, hash_type;

        switch (type) {
        case ICE_VSI_PF:
                lut_type = ICE_AQ_VSI_Q_OPT_RSS_LUT_PF;
                hash_type = ICE_AQ_VSI_Q_OPT_RSS_TPLZ;
                break;
        case ICE_VSI_VF:
        case ICE_VSI_VMDQ2:
                lut_type = ICE_AQ_VSI_Q_OPT_RSS_LUT_VSI;
                hash_type = ICE_AQ_VSI_Q_OPT_RSS_TPLZ;
                break;
        default:
                /* Other VSI types do not support RSS */
                return;
        }

        ctx->info.q_opt_rss = (((lut_type << ICE_AQ_VSI_Q_OPT_RSS_LUT_S) &
                                 ICE_AQ_VSI_Q_OPT_RSS_LUT_M) |
                                ((hash_type << ICE_AQ_VSI_Q_OPT_RSS_HASH_S) &
                                 ICE_AQ_VSI_Q_OPT_RSS_HASH_M));
}

/**
 * ice_setup_vsi_qmap - Setup the queue mapping for a VSI
 * @vsi: the VSI to configure
 * @ctx: the VSI context to configure
 *
 * Configures the context for the given VSI, setting up how the firmware
 * should map the queues for this VSI.
 *
 * @pre vsi->qmap_type is set to a valid type
 */
static int
ice_setup_vsi_qmap(struct ice_vsi *vsi, struct ice_vsi_ctx *ctx)
{
        int pow = 0;
        u16 qmap;

        MPASS(vsi->rx_qmap != NULL);

        switch (vsi->qmap_type) {
        case ICE_RESMGR_ALLOC_CONTIGUOUS:
                ctx->info.mapping_flags |= CPU_TO_LE16(ICE_AQ_VSI_Q_MAP_CONTIG);

                ctx->info.q_mapping[0] = CPU_TO_LE16(vsi->rx_qmap[0]);
                ctx->info.q_mapping[1] = CPU_TO_LE16(vsi->num_rx_queues);

                break;
        case ICE_RESMGR_ALLOC_SCATTERED:
                ctx->info.mapping_flags |= CPU_TO_LE16(ICE_AQ_VSI_Q_MAP_NONCONTIG);

                for (int i = 0; i < vsi->num_rx_queues; i++)
                        ctx->info.q_mapping[i] = CPU_TO_LE16(vsi->rx_qmap[i]);
                break;
        default:
                return (EOPNOTSUPP);
        }

        /* Calculate the next power-of-2 of number of queues */
        if (vsi->num_rx_queues)
                pow = flsl(vsi->num_rx_queues - 1);

        /* Assign all the queues to traffic class zero */
        qmap = (pow << ICE_AQ_VSI_TC_Q_NUM_S) & ICE_AQ_VSI_TC_Q_NUM_M;
        ctx->info.tc_mapping[0] = CPU_TO_LE16(qmap);

        /* Fill out default driver TC queue info for VSI */
        vsi->tc_info[0].qoffset = 0;
        vsi->tc_info[0].qcount_rx = vsi->num_rx_queues;
        vsi->tc_info[0].qcount_tx = vsi->num_tx_queues;
        for (int i = 1; i < ICE_MAX_TRAFFIC_CLASS; i++) {
                vsi->tc_info[i].qoffset = 0;
                vsi->tc_info[i].qcount_rx = 1;
                vsi->tc_info[i].qcount_tx = 1;
        }
        vsi->tc_map = 0x1;

        return 0;
}

/**
 * ice_setup_vsi_mirroring -- Setup a VSI for mirroring PF VSI traffic
 * @vsi: VSI to setup
 *
 * @pre vsi->mirror_src_vsi is set to the SW VSI num that traffic is to be
 * mirrored from
 *
 * Returns 0 on success, EINVAL on failure.
 */
int
ice_setup_vsi_mirroring(struct ice_vsi *vsi)
{
        struct ice_mir_rule_buf rule = { };
        struct ice_softc *sc = vsi->sc;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;
        u16 rule_id, dest_vsi;
        u16 count = 1;

        rule.vsi_idx = ice_get_hw_vsi_num(hw, vsi->mirror_src_vsi);
        rule.add = true;

        dest_vsi = ice_get_hw_vsi_num(hw, vsi->idx);
        rule_id = ICE_INVAL_MIRROR_RULE_ID;
        status = ice_aq_add_update_mir_rule(hw, ICE_AQC_RULE_TYPE_VPORT_INGRESS,
                                            dest_vsi, count, &rule, NULL,
                                            &rule_id);
        if (status) {
                device_printf(dev,
                    "Could not add INGRESS rule for mirror vsi %d to vsi %d, err %s aq_err %s\n",
                    rule.vsi_idx, dest_vsi, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EINVAL);
        }

        vsi->rule_mir_ingress = rule_id;

        rule_id = ICE_INVAL_MIRROR_RULE_ID;
        status = ice_aq_add_update_mir_rule(hw, ICE_AQC_RULE_TYPE_VPORT_EGRESS,
                                            dest_vsi, count, &rule, NULL, &rule_id);
        if (status) {
                device_printf(dev,
                    "Could not add EGRESS rule for mirror vsi %d to vsi %d, err %s aq_err %s\n",
                    rule.vsi_idx, dest_vsi, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EINVAL);
        }

        vsi->rule_mir_egress = rule_id;

        return (0);
}

/**
 * ice_remove_vsi_mirroring -- Teardown any VSI mirroring rules
 * @vsi: VSI to remove mirror rules from
 */
static void
ice_remove_vsi_mirroring(struct ice_vsi *vsi)
{
        struct ice_hw *hw = &vsi->sc->hw;
        int status = 0;
        bool keep_alloc = false;

        if (vsi->rule_mir_ingress != ICE_INVAL_MIRROR_RULE_ID)
                status = ice_aq_delete_mir_rule(hw, vsi->rule_mir_ingress, keep_alloc, NULL);

        if (status)
                device_printf(vsi->sc->dev, "Could not remove mirror VSI ingress rule, err %s aq_err %s\n",
                              ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));

        status = 0;

        if (vsi->rule_mir_egress != ICE_INVAL_MIRROR_RULE_ID)
                status = ice_aq_delete_mir_rule(hw, vsi->rule_mir_egress, keep_alloc, NULL);

        if (status)
                device_printf(vsi->sc->dev, "Could not remove mirror VSI egress rule, err %s aq_err %s\n",
                              ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
}

/**
 * ice_initialize_vsi - Initialize a VSI for use
 * @vsi: the vsi to initialize
 *
 * Initialize a VSI over the adminq and prepare it for operation.
 *
 * @pre vsi->num_tx_queues is set
 * @pre vsi->num_rx_queues is set
 */
int
ice_initialize_vsi(struct ice_vsi *vsi)
{
        struct ice_vsi_ctx ctx = { 0 };
        struct ice_hw *hw = &vsi->sc->hw;
        u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
        int status;
        int err;

        /* For now, we only have code supporting PF VSIs */
        switch (vsi->type) {
        case ICE_VSI_PF:
                ctx.flags = ICE_AQ_VSI_TYPE_PF;
                break;
        case ICE_VSI_VMDQ2:
                ctx.flags = ICE_AQ_VSI_TYPE_VMDQ2;
                break;
#ifdef PCI_IOV
        case ICE_VSI_VF:
                ctx.flags = ICE_AQ_VSI_TYPE_VF;
                ctx.vf_num = vsi->vf_num;
                break;
#endif
        default:
                return (ENODEV);
        }

        ice_set_default_vsi_ctx(&ctx);
        ice_set_rss_vsi_ctx(&ctx, vsi->type);

        /* XXX: VSIs of other types may need different port info? */
        ctx.info.sw_id = hw->port_info->sw_id;

        /* Set some RSS parameters based on the VSI type */
        ice_vsi_set_rss_params(vsi);

        /* Initialize the Rx queue mapping for this VSI */
        err = ice_setup_vsi_qmap(vsi, &ctx);
        if (err) {
                return err;
        }

        /* (Re-)add VSI to HW VSI handle list */
        status = ice_add_vsi(hw, vsi->idx, &ctx, NULL);
        if (status != 0) {
                device_printf(vsi->sc->dev,
                    "Add VSI AQ call failed, err %s aq_err %s\n",
                    ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }
        vsi->info = ctx.info;

        /* Initialize VSI with just 1 TC to start */
        max_txqs[0] = vsi->num_tx_queues;

        status = ice_cfg_vsi_lan(hw->port_info, vsi->idx,
                              ICE_DFLT_TRAFFIC_CLASS, max_txqs);
        if (status) {
                device_printf(vsi->sc->dev,
                    "Failed VSI lan queue config, err %s aq_err %s\n",
                    ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                ice_deinit_vsi(vsi);
                return (ENODEV);
        }

        /* Reset VSI stats */
        ice_reset_vsi_stats(vsi);

        return 0;
}

/**
 * ice_deinit_vsi - Tell firmware to release resources for a VSI
 * @vsi: the VSI to release
 *
 * Helper function which requests the firmware to release the hardware
 * resources associated with a given VSI.
 */
void
ice_deinit_vsi(struct ice_vsi *vsi)
{
        struct ice_vsi_ctx ctx = { 0 };
        struct ice_softc *sc = vsi->sc;
        struct ice_hw *hw = &sc->hw;
        int status;

        /* Assert that the VSI pointer matches in the list */
        MPASS(vsi == sc->all_vsi[vsi->idx]);

        ctx.info = vsi->info;

        status = ice_rm_vsi_lan_cfg(hw->port_info, vsi->idx);
        if (status) {
                /*
                 * This should only fail if the VSI handle is invalid, or if
                 * any of the nodes have leaf nodes which are still in use.
                 */
                device_printf(sc->dev,
                              "Unable to remove scheduler nodes for VSI %d, err %s\n",
                              vsi->idx, ice_status_str(status));
        }

        /* Tell firmware to release the VSI resources */
        status = ice_free_vsi(hw, vsi->idx, &ctx, false, NULL);
        if (status != 0) {
                device_printf(sc->dev,
                    "Free VSI %u AQ call failed, err %s aq_err %s\n",
                    vsi->idx, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
        }
}

/**
 * ice_release_vsi - Release resources associated with a VSI
 * @vsi: the VSI to release
 *
 * Release software and firmware resources associated with a VSI. Release the
 * queue managers associated with this VSI. Also free the VSI structure memory
 * if the VSI was allocated dynamically using ice_alloc_vsi().
 */
void
ice_release_vsi(struct ice_vsi *vsi)
{
        struct ice_softc *sc = vsi->sc;
        int idx = vsi->idx;

        /* Assert that the VSI pointer matches in the list */
        MPASS(vsi == sc->all_vsi[idx]);

        /* Cleanup RSS configuration */
        if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_RSS))
                ice_clean_vsi_rss_cfg(vsi);

        ice_del_vsi_sysctl_ctx(vsi);

        /* Remove the configured mirror rule, if it exists */
        ice_remove_vsi_mirroring(vsi);

        /*
         * If we unload the driver after a reset fails, we do not need to do
         * this step.
         */
        if (!ice_test_state(&sc->state, ICE_STATE_RESET_FAILED))
                ice_deinit_vsi(vsi);

        ice_free_vsi_qmaps(vsi);

        if (vsi->dynamic) {
                free(sc->all_vsi[idx], M_ICE);
        }

        sc->all_vsi[idx] = NULL;
}

/**
 * ice_aq_speed_to_rate - Convert AdminQ speed enum to baudrate
 * @pi: port info data
 *
 * Returns the baudrate value for the current link speed of a given port.
 */
uint64_t
ice_aq_speed_to_rate(struct ice_port_info *pi)
{
        switch (pi->phy.link_info.link_speed) {
        case ICE_AQ_LINK_SPEED_200GB:
                return IF_Gbps(200);
        case ICE_AQ_LINK_SPEED_100GB:
                return IF_Gbps(100);
        case ICE_AQ_LINK_SPEED_50GB:
                return IF_Gbps(50);
        case ICE_AQ_LINK_SPEED_40GB:
                return IF_Gbps(40);
        case ICE_AQ_LINK_SPEED_25GB:
                return IF_Gbps(25);
        case ICE_AQ_LINK_SPEED_10GB:
                return IF_Gbps(10);
        case ICE_AQ_LINK_SPEED_5GB:
                return IF_Gbps(5);
        case ICE_AQ_LINK_SPEED_2500MB:
                return IF_Mbps(2500);
        case ICE_AQ_LINK_SPEED_1000MB:
                return IF_Mbps(1000);
        case ICE_AQ_LINK_SPEED_100MB:
                return IF_Mbps(100);
        case ICE_AQ_LINK_SPEED_10MB:
                return IF_Mbps(10);
        case ICE_AQ_LINK_SPEED_UNKNOWN:
        default:
                /* return 0 if we don't know the link speed */
                return 0;
        }
}

/**
 * ice_aq_speed_to_str - Convert AdminQ speed enum to string representation
 * @pi: port info data
 *
 * Returns the string representation of the current link speed for a given
 * port.
 */
static const char *
ice_aq_speed_to_str(struct ice_port_info *pi)
{
        switch (pi->phy.link_info.link_speed) {
        case ICE_AQ_LINK_SPEED_200GB:
                return "200 Gbps";
        case ICE_AQ_LINK_SPEED_100GB:
                return "100 Gbps";
        case ICE_AQ_LINK_SPEED_50GB:
                return "50 Gbps";
        case ICE_AQ_LINK_SPEED_40GB:
                return "40 Gbps";
        case ICE_AQ_LINK_SPEED_25GB:
                return "25 Gbps";
        case ICE_AQ_LINK_SPEED_20GB:
                return "20 Gbps";
        case ICE_AQ_LINK_SPEED_10GB:
                return "10 Gbps";
        case ICE_AQ_LINK_SPEED_5GB:
                return "5 Gbps";
        case ICE_AQ_LINK_SPEED_2500MB:
                return "2.5 Gbps";
        case ICE_AQ_LINK_SPEED_1000MB:
                return "1 Gbps";
        case ICE_AQ_LINK_SPEED_100MB:
                return "100 Mbps";
        case ICE_AQ_LINK_SPEED_10MB:
                return "10 Mbps";
        case ICE_AQ_LINK_SPEED_UNKNOWN:
        default:
                return "Unknown speed";
        }
}

/**
 * ice_get_phy_type_low - Get media associated with phy_type_low
 * @phy_type_low: the low 64bits of phy_type from the AdminQ
 *
 * Given the lower 64bits of the phy_type from the hardware, return the
 * ifm_active bit associated. Return IFM_UNKNOWN when phy_type_low is unknown.
 * Note that only one of ice_get_phy_type_low or ice_get_phy_type_high should
 * be called. If phy_type_low is zero, call ice_phy_type_high.
 */
int
ice_get_phy_type_low(uint64_t phy_type_low)
{
        switch (phy_type_low) {
        case ICE_PHY_TYPE_LOW_100BASE_TX:
                return IFM_100_TX;
        case ICE_PHY_TYPE_LOW_100M_SGMII:
                return IFM_100_SGMII;
        case ICE_PHY_TYPE_LOW_1000BASE_T:
                return IFM_1000_T;
        case ICE_PHY_TYPE_LOW_1000BASE_SX:
                return IFM_1000_SX;
        case ICE_PHY_TYPE_LOW_1000BASE_LX:
                return IFM_1000_LX;
        case ICE_PHY_TYPE_LOW_1000BASE_KX:
                return IFM_1000_KX;
        case ICE_PHY_TYPE_LOW_1G_SGMII:
                return IFM_1000_SGMII;
        case ICE_PHY_TYPE_LOW_2500BASE_T:
                return IFM_2500_T;
        case ICE_PHY_TYPE_LOW_2500BASE_X:
                return IFM_2500_X;
        case ICE_PHY_TYPE_LOW_2500BASE_KX:
                return IFM_2500_KX;
        case ICE_PHY_TYPE_LOW_5GBASE_T:
                return IFM_5000_T;
        case ICE_PHY_TYPE_LOW_5GBASE_KR:
                return IFM_5000_KR;
        case ICE_PHY_TYPE_LOW_10GBASE_T:
                return IFM_10G_T;
        case ICE_PHY_TYPE_LOW_10G_SFI_DA:
                return IFM_10G_TWINAX;
        case ICE_PHY_TYPE_LOW_10GBASE_SR:
                return IFM_10G_SR;
        case ICE_PHY_TYPE_LOW_10GBASE_LR:
                return IFM_10G_LR;
        case ICE_PHY_TYPE_LOW_10GBASE_KR_CR1:
                return IFM_10G_KR;
        case ICE_PHY_TYPE_LOW_10G_SFI_AOC_ACC:
                return IFM_10G_AOC;
        case ICE_PHY_TYPE_LOW_10G_SFI_C2C:
                return IFM_10G_SFI;
        case ICE_PHY_TYPE_LOW_25GBASE_T:
                return IFM_25G_T;
        case ICE_PHY_TYPE_LOW_25GBASE_CR:
                return IFM_25G_CR;
        case ICE_PHY_TYPE_LOW_25GBASE_CR_S:
                return IFM_25G_CR_S;
        case ICE_PHY_TYPE_LOW_25GBASE_CR1:
                return IFM_25G_CR1;
        case ICE_PHY_TYPE_LOW_25GBASE_SR:
                return IFM_25G_SR;
        case ICE_PHY_TYPE_LOW_25GBASE_LR:
                return IFM_25G_LR;
        case ICE_PHY_TYPE_LOW_25GBASE_KR:
                return IFM_25G_KR;
        case ICE_PHY_TYPE_LOW_25GBASE_KR_S:
                return IFM_25G_KR_S;
        case ICE_PHY_TYPE_LOW_25GBASE_KR1:
                return IFM_25G_KR1;
        case ICE_PHY_TYPE_LOW_25G_AUI_AOC_ACC:
                return IFM_25G_AOC;
        case ICE_PHY_TYPE_LOW_25G_AUI_C2C:
                return IFM_25G_AUI;
        case ICE_PHY_TYPE_LOW_40GBASE_CR4:
                return IFM_40G_CR4;
        case ICE_PHY_TYPE_LOW_40GBASE_SR4:
                return IFM_40G_SR4;
        case ICE_PHY_TYPE_LOW_40GBASE_LR4:
                return IFM_40G_LR4;
        case ICE_PHY_TYPE_LOW_40GBASE_KR4:
                return IFM_40G_KR4;
        case ICE_PHY_TYPE_LOW_40G_XLAUI_AOC_ACC:
                return IFM_40G_XLAUI_AC;
        case ICE_PHY_TYPE_LOW_40G_XLAUI:
                return IFM_40G_XLAUI;
        case ICE_PHY_TYPE_LOW_50GBASE_CR2:
                return IFM_50G_CR2;
        case ICE_PHY_TYPE_LOW_50GBASE_SR2:
                return IFM_50G_SR2;
        case ICE_PHY_TYPE_LOW_50GBASE_LR2:
                return IFM_50G_LR2;
        case ICE_PHY_TYPE_LOW_50GBASE_KR2:
                return IFM_50G_KR2;
        case ICE_PHY_TYPE_LOW_50G_LAUI2_AOC_ACC:
                return IFM_50G_LAUI2_AC;
        case ICE_PHY_TYPE_LOW_50G_LAUI2:
                return IFM_50G_LAUI2;
        case ICE_PHY_TYPE_LOW_50G_AUI2_AOC_ACC:
                return IFM_50G_AUI2_AC;
        case ICE_PHY_TYPE_LOW_50G_AUI2:
                return IFM_50G_AUI2;
        case ICE_PHY_TYPE_LOW_50GBASE_CP:
                return IFM_50G_CP;
        case ICE_PHY_TYPE_LOW_50GBASE_SR:
                return IFM_50G_SR;
        case ICE_PHY_TYPE_LOW_50GBASE_FR:
                return IFM_50G_FR;
        case ICE_PHY_TYPE_LOW_50GBASE_LR:
                return IFM_50G_LR;
        case ICE_PHY_TYPE_LOW_50GBASE_KR_PAM4:
                return IFM_50G_KR_PAM4;
        case ICE_PHY_TYPE_LOW_50G_AUI1_AOC_ACC:
                return IFM_50G_AUI1_AC;
        case ICE_PHY_TYPE_LOW_50G_AUI1:
                return IFM_50G_AUI1;
        case ICE_PHY_TYPE_LOW_100GBASE_CR4:
                return IFM_100G_CR4;
        case ICE_PHY_TYPE_LOW_100GBASE_SR4:
                return IFM_100G_SR4;
        case ICE_PHY_TYPE_LOW_100GBASE_LR4:
                return IFM_100G_LR4;
        case ICE_PHY_TYPE_LOW_100GBASE_KR4:
                return IFM_100G_KR4;
        case ICE_PHY_TYPE_LOW_100G_CAUI4_AOC_ACC:
                return IFM_100G_CAUI4_AC;
        case ICE_PHY_TYPE_LOW_100G_CAUI4:
                return IFM_100G_CAUI4;
        case ICE_PHY_TYPE_LOW_100G_AUI4_AOC_ACC:
                return IFM_100G_AUI4_AC;
        case ICE_PHY_TYPE_LOW_100G_AUI4:
                return IFM_100G_AUI4;
        case ICE_PHY_TYPE_LOW_100GBASE_CR_PAM4:
                return IFM_100G_CR_PAM4;
        case ICE_PHY_TYPE_LOW_100GBASE_KR_PAM4:
                return IFM_100G_KR_PAM4;
        case ICE_PHY_TYPE_LOW_100GBASE_CP2:
                return IFM_100G_CP2;
        case ICE_PHY_TYPE_LOW_100GBASE_SR2:
                return IFM_100G_SR2;
        case ICE_PHY_TYPE_LOW_100GBASE_DR:
                return IFM_100G_DR;
        default:
                return IFM_UNKNOWN;
        }
}

/**
 * ice_get_phy_type_high - Get media associated with phy_type_high
 * @phy_type_high: the upper 64bits of phy_type from the AdminQ
 *
 * Given the upper 64bits of the phy_type from the hardware, return the
 * ifm_active bit associated. Return IFM_UNKNOWN on an unknown value. Note
 * that only one of ice_get_phy_type_low or ice_get_phy_type_high should be
 * called. If phy_type_high is zero, call ice_get_phy_type_low.
 */
int
ice_get_phy_type_high(uint64_t phy_type_high)
{
        switch (phy_type_high) {
        case ICE_PHY_TYPE_HIGH_100GBASE_KR2_PAM4:
                return IFM_100G_KR2_PAM4;
        case ICE_PHY_TYPE_HIGH_100G_CAUI2_AOC_ACC:
                return IFM_100G_CAUI2_AC;
        case ICE_PHY_TYPE_HIGH_100G_CAUI2:
                return IFM_100G_CAUI2;
        case ICE_PHY_TYPE_HIGH_100G_AUI2_AOC_ACC:
                return IFM_100G_AUI2_AC;
        case ICE_PHY_TYPE_HIGH_100G_AUI2:
                return IFM_100G_AUI2;
        case ICE_PHY_TYPE_HIGH_200G_CR4_PAM4:
                return IFM_200G_CR4_PAM4;
        case ICE_PHY_TYPE_HIGH_200G_SR4:
                return IFM_200G_SR4;
        case ICE_PHY_TYPE_HIGH_200G_FR4:
                return IFM_200G_FR4;
        case ICE_PHY_TYPE_HIGH_200G_LR4:
                return IFM_200G_LR4;
        case ICE_PHY_TYPE_HIGH_200G_DR4:
                return IFM_200G_DR4;
        case ICE_PHY_TYPE_HIGH_200G_KR4_PAM4:
                return IFM_200G_KR4_PAM4;
        case ICE_PHY_TYPE_HIGH_200G_AUI4_AOC_ACC:
                return IFM_200G_AUI4_AC;
        case ICE_PHY_TYPE_HIGH_200G_AUI4:
                return IFM_200G_AUI4;
        case ICE_PHY_TYPE_HIGH_200G_AUI8_AOC_ACC:
                return IFM_200G_AUI8_AC;
        case ICE_PHY_TYPE_HIGH_200G_AUI8:
                return IFM_200G_AUI8;
        default:
                return IFM_UNKNOWN;
        }
}

/**
 * ice_phy_types_to_max_rate - Returns port's max supported baudrate
 * @pi: port info struct
 *
 * ice_aq_get_phy_caps() w/ ICE_AQC_REPORT_TOPO_CAP_MEDIA parameter needs
 * to have been called before this function for it to work.
 */
static uint64_t
ice_phy_types_to_max_rate(struct ice_port_info *pi)
{
        uint64_t phy_low = pi->phy.phy_type_low;
        uint64_t phy_high = pi->phy.phy_type_high;
        uint64_t max_rate = 0;
        int bit;

        /*
         * These are based on the indices used in the BIT() macros for
         * ICE_PHY_TYPE_LOW_*
         */
        static const uint64_t phy_rates[] = {
            IF_Mbps(100),
            IF_Mbps(100),
            IF_Gbps(1ULL),
            IF_Gbps(1ULL),
            IF_Gbps(1ULL),
            IF_Gbps(1ULL),
            IF_Gbps(1ULL),
            IF_Mbps(2500ULL),
            IF_Mbps(2500ULL),
            IF_Mbps(2500ULL),
            IF_Gbps(5ULL),
            IF_Gbps(5ULL),
            IF_Gbps(10ULL),
            IF_Gbps(10ULL),
            IF_Gbps(10ULL),
            IF_Gbps(10ULL),
            IF_Gbps(10ULL),
            IF_Gbps(10ULL),
            IF_Gbps(10ULL),
            IF_Gbps(25ULL),
            IF_Gbps(25ULL),
            IF_Gbps(25ULL),
            IF_Gbps(25ULL),
            IF_Gbps(25ULL),
            IF_Gbps(25ULL),
            IF_Gbps(25ULL),
            IF_Gbps(25ULL),
            IF_Gbps(25ULL),
            IF_Gbps(25ULL),
            IF_Gbps(25ULL),
            IF_Gbps(40ULL),
            IF_Gbps(40ULL),
            IF_Gbps(40ULL),
            IF_Gbps(40ULL),
            IF_Gbps(40ULL),
            IF_Gbps(40ULL),
            IF_Gbps(50ULL),
            IF_Gbps(50ULL),
            IF_Gbps(50ULL),
            IF_Gbps(50ULL),
            IF_Gbps(50ULL),
            IF_Gbps(50ULL),
            IF_Gbps(50ULL),
            IF_Gbps(50ULL),
            IF_Gbps(50ULL),
            IF_Gbps(50ULL),
            IF_Gbps(50ULL),
            IF_Gbps(50ULL),
            IF_Gbps(50ULL),
            IF_Gbps(50ULL),
            IF_Gbps(50ULL),
            IF_Gbps(100ULL),
            IF_Gbps(100ULL),
            IF_Gbps(100ULL),
            IF_Gbps(100ULL),
            IF_Gbps(100ULL),
            IF_Gbps(100ULL),
            IF_Gbps(100ULL),
            IF_Gbps(100ULL),
            IF_Gbps(100ULL),
            IF_Gbps(100ULL),
            IF_Gbps(100ULL),
            IF_Gbps(100ULL),
            IF_Gbps(100ULL),
            /* These rates are for ICE_PHY_TYPE_HIGH_* */
            IF_Gbps(100ULL),
            IF_Gbps(100ULL),
            IF_Gbps(100ULL),
            IF_Gbps(100ULL),
            IF_Gbps(100ULL),
            IF_Gbps(200ULL),
            IF_Gbps(200ULL),
            IF_Gbps(200ULL),
            IF_Gbps(200ULL),
            IF_Gbps(200ULL),
            IF_Gbps(200ULL),
            IF_Gbps(200ULL),
            IF_Gbps(200ULL),
            IF_Gbps(200ULL),
            IF_Gbps(200ULL),
        };

        /* coverity[address_of] */
        for_each_set_bit(bit, &phy_high, 64)
                if ((bit + 64) < (int)ARRAY_SIZE(phy_rates))
                        max_rate = uqmax(max_rate, phy_rates[(bit + 64)]);

        /* coverity[address_of] */
        for_each_set_bit(bit, &phy_low, 64)
                max_rate = uqmax(max_rate, phy_rates[bit]);

        return (max_rate);
}

/* The if_media type is split over the original 5 bit media variant field,
 * along with extended types using up extra bits in the options section.
 * We want to convert this split number into a bitmap index, so we reverse the
 * calculation of IFM_X here.
 */
#define IFM_IDX(x) (((x) & IFM_TMASK) | \
                    (((x) & IFM_ETH_XTYPE) >> IFM_ETH_XSHIFT))

/**
 * ice_add_media_types - Add supported media types to the media structure
 * @sc: ice private softc structure
 * @media: ifmedia structure to setup
 *
 * Looks up the supported phy types, and initializes the various media types
 * available.
 *
 * @pre this function must be protected from being called while another thread
 * is accessing the ifmedia types.
 */
int
ice_add_media_types(struct ice_softc *sc, struct ifmedia *media)
{
        struct ice_aqc_get_phy_caps_data pcaps = { 0 };
        struct ice_port_info *pi = sc->hw.port_info;
        int status;
        uint64_t phy_low, phy_high;
        int bit;

        ASSERT_CFG_LOCKED(sc);

        /* the maximum possible media type index is 511. We probably don't
         * need most of this space, but this ensures future compatibility when
         * additional media types are used.
         */
        ice_declare_bitmap(already_added, 511);

        /* Remove all previous media types */
        ifmedia_removeall(media);

        status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG,
                                     &pcaps, NULL);
        if (status) {
                device_printf(sc->dev,
                    "%s: ice_aq_get_phy_caps (ACTIVE) failed; status %s, aq_err %s\n",
                    __func__, ice_status_str(status),
                    ice_aq_str(sc->hw.adminq.sq_last_status));
                return (status);
        }
        phy_low = le64toh(pcaps.phy_type_low);
        phy_high = le64toh(pcaps.phy_type_high);

        /* make sure the added bitmap is zero'd */
        memset(already_added, 0, sizeof(already_added));

        /* coverity[address_of] */
        for_each_set_bit(bit, &phy_low, 64) {
                uint64_t type = BIT_ULL(bit);
                int ostype;

                /* get the OS media type */
                ostype = ice_get_phy_type_low(type);

                /* don't bother adding the unknown type */
                if (ostype == IFM_UNKNOWN)
                        continue;

                /* only add each media type to the list once */
                if (ice_is_bit_set(already_added, IFM_IDX(ostype)))
                        continue;

                ifmedia_add(media, IFM_ETHER | ostype, 0, NULL);
                ice_set_bit(IFM_IDX(ostype), already_added);
        }

        /* coverity[address_of] */
        for_each_set_bit(bit, &phy_high, 64) {
                uint64_t type = BIT_ULL(bit);
                int ostype;

                /* get the OS media type */
                ostype = ice_get_phy_type_high(type);

                /* don't bother adding the unknown type */
                if (ostype == IFM_UNKNOWN)
                        continue;

                /* only add each media type to the list once */
                if (ice_is_bit_set(already_added, IFM_IDX(ostype)))
                        continue;

                ifmedia_add(media, IFM_ETHER | ostype, 0, NULL);
                ice_set_bit(IFM_IDX(ostype), already_added);
        }

        /* Use autoselect media by default */
        ifmedia_add(media, IFM_ETHER | IFM_AUTO, 0, NULL);
        ifmedia_set(media, IFM_ETHER | IFM_AUTO);

        return (0);
}

/**
 * ice_configure_rxq_interrupt - Configure HW Rx queue for an MSI-X interrupt
 * @hw: ice hw structure
 * @rxqid: Rx queue index in PF space
 * @vector: MSI-X vector index in PF/VF space
 * @itr_idx: ITR index to use for interrupt
 *
 * @remark ice_flush() may need to be called after this
 */
void
ice_configure_rxq_interrupt(struct ice_hw *hw, u16 rxqid, u16 vector, u8 itr_idx)
{
        u32 val;

        MPASS(itr_idx <= ICE_ITR_NONE);

        val = (QINT_RQCTL_CAUSE_ENA_M |
               (itr_idx << QINT_RQCTL_ITR_INDX_S) |
               (vector << QINT_RQCTL_MSIX_INDX_S));
        wr32(hw, QINT_RQCTL(rxqid), val);
}

/**
 * ice_configure_all_rxq_interrupts - Configure HW Rx queues for MSI-X interrupts
 * @vsi: the VSI to configure
 *
 * Called when setting up MSI-X interrupts to configure the Rx hardware queues.
 */
void
ice_configure_all_rxq_interrupts(struct ice_vsi *vsi)
{
        struct ice_hw *hw = &vsi->sc->hw;
        int i;

        for (i = 0; i < vsi->num_rx_queues; i++) {
                struct ice_rx_queue *rxq = &vsi->rx_queues[i];

                ice_configure_rxq_interrupt(hw, vsi->rx_qmap[rxq->me],
                                            rxq->irqv->me, ICE_RX_ITR);

                ice_debug(hw, ICE_DBG_INIT,
                    "RXQ(%d) intr enable: me %d rxqid %d vector %d\n",
                    i, rxq->me, vsi->rx_qmap[rxq->me], rxq->irqv->me);
        }

        ice_flush(hw);
}

/**
 * ice_configure_txq_interrupt - Configure HW Tx queue for an MSI-X interrupt
 * @hw: ice hw structure
 * @txqid: Tx queue index in PF space
 * @vector: MSI-X vector index in PF/VF space
 * @itr_idx: ITR index to use for interrupt
 *
 * @remark ice_flush() may need to be called after this
 */
void
ice_configure_txq_interrupt(struct ice_hw *hw, u16 txqid, u16 vector, u8 itr_idx)
{
        u32 val;

        MPASS(itr_idx <= ICE_ITR_NONE);

        val = (QINT_TQCTL_CAUSE_ENA_M |
               (itr_idx << QINT_TQCTL_ITR_INDX_S) |
               (vector << QINT_TQCTL_MSIX_INDX_S));
        wr32(hw, QINT_TQCTL(txqid), val);
}

/**
 * ice_configure_all_txq_interrupts - Configure HW Tx queues for MSI-X interrupts
 * @vsi: the VSI to configure
 *
 * Called when setting up MSI-X interrupts to configure the Tx hardware queues.
 */
void
ice_configure_all_txq_interrupts(struct ice_vsi *vsi)
{
        struct ice_hw *hw = &vsi->sc->hw;
        int i;

        for (i = 0; i < vsi->num_tx_queues; i++) {
                struct ice_tx_queue *txq = &vsi->tx_queues[i];

                ice_configure_txq_interrupt(hw, vsi->tx_qmap[txq->me],
                                            txq->irqv->me, ICE_TX_ITR);
        }

        ice_flush(hw);
}

/**
 * ice_flush_rxq_interrupts - Unconfigure Hw Rx queues MSI-X interrupt cause
 * @vsi: the VSI to configure
 *
 * Unset the CAUSE_ENA flag of the TQCTL register for each queue, then trigger
 * a software interrupt on that cause. This is required as part of the Rx
 * queue disable logic to dissociate the Rx queue from the interrupt.
 *
 * Note: this function must be called prior to disabling Rx queues with
 * ice_control_all_rx_queues, otherwise the Rx queue may not be disabled properly.
 */
void
ice_flush_rxq_interrupts(struct ice_vsi *vsi)
{
        struct ice_hw *hw = &vsi->sc->hw;
        int i;

        for (i = 0; i < vsi->num_rx_queues; i++) {
                struct ice_rx_queue *rxq = &vsi->rx_queues[i];
                u32 reg, val;

                /* Clear the CAUSE_ENA flag */
                reg = vsi->rx_qmap[rxq->me];
                val = rd32(hw, QINT_RQCTL(reg));
                val &= ~QINT_RQCTL_CAUSE_ENA_M;
                wr32(hw, QINT_RQCTL(reg), val);

                ice_flush(hw);

                /* Trigger a software interrupt to complete interrupt
                 * dissociation.
                 */
                wr32(hw, GLINT_DYN_CTL(rxq->irqv->me),
                     GLINT_DYN_CTL_SWINT_TRIG_M | GLINT_DYN_CTL_INTENA_MSK_M);
        }
}

/**
 * ice_flush_txq_interrupts - Unconfigure Hw Tx queues MSI-X interrupt cause
 * @vsi: the VSI to configure
 *
 * Unset the CAUSE_ENA flag of the TQCTL register for each queue, then trigger
 * a software interrupt on that cause. This is required as part of the Tx
 * queue disable logic to dissociate the Tx queue from the interrupt.
 *
 * Note: this function must be called prior to ice_vsi_disable_tx, otherwise
 * the Tx queue disable may not complete properly.
 */
void
ice_flush_txq_interrupts(struct ice_vsi *vsi)
{
        struct ice_hw *hw = &vsi->sc->hw;
        int i;

        for (i = 0; i < vsi->num_tx_queues; i++) {
                struct ice_tx_queue *txq = &vsi->tx_queues[i];
                u32 reg, val;

                /* Clear the CAUSE_ENA flag */
                reg = vsi->tx_qmap[txq->me];
                val = rd32(hw, QINT_TQCTL(reg));
                val &= ~QINT_TQCTL_CAUSE_ENA_M;
                wr32(hw, QINT_TQCTL(reg), val);

                ice_flush(hw);

                /* Trigger a software interrupt to complete interrupt
                 * dissociation.
                 */
                wr32(hw, GLINT_DYN_CTL(txq->irqv->me),
                     GLINT_DYN_CTL_SWINT_TRIG_M | GLINT_DYN_CTL_INTENA_MSK_M);
        }
}

/**
 * ice_configure_rx_itr - Configure the Rx ITR settings for this VSI
 * @vsi: the VSI to configure
 *
 * Program the hardware ITR registers with the settings for this VSI.
 */
void
ice_configure_rx_itr(struct ice_vsi *vsi)
{
        struct ice_hw *hw = &vsi->sc->hw;
        int i;

        /* TODO: Handle per-queue/per-vector ITR? */

        for (i = 0; i < vsi->num_rx_queues; i++) {
                struct ice_rx_queue *rxq = &vsi->rx_queues[i];

                wr32(hw, GLINT_ITR(ICE_RX_ITR, rxq->irqv->me),
                     ice_itr_to_reg(hw, vsi->rx_itr));
        }

        ice_flush(hw);
}

/**
 * ice_configure_tx_itr - Configure the Tx ITR settings for this VSI
 * @vsi: the VSI to configure
 *
 * Program the hardware ITR registers with the settings for this VSI.
 */
void
ice_configure_tx_itr(struct ice_vsi *vsi)
{
        struct ice_hw *hw = &vsi->sc->hw;
        int i;

        /* TODO: Handle per-queue/per-vector ITR? */

        for (i = 0; i < vsi->num_tx_queues; i++) {
                struct ice_tx_queue *txq = &vsi->tx_queues[i];

                wr32(hw, GLINT_ITR(ICE_TX_ITR, txq->irqv->me),
                     ice_itr_to_reg(hw, vsi->tx_itr));
        }

        ice_flush(hw);
}

/**
 * ice_setup_tx_ctx - Setup an ice_tlan_ctx structure for a queue
 * @txq: the Tx queue to configure
 * @tlan_ctx: the Tx LAN queue context structure to initialize
 * @pf_q: real queue number
 */
static int
ice_setup_tx_ctx(struct ice_tx_queue *txq, struct ice_tlan_ctx *tlan_ctx, u16 pf_q)
{
        struct ice_vsi *vsi = txq->vsi;
        struct ice_softc *sc = vsi->sc;
        struct ice_hw *hw = &sc->hw;

        tlan_ctx->port_num = hw->port_info->lport;

        /* number of descriptors in the queue */
        tlan_ctx->qlen = txq->desc_count;

        /* set the transmit queue base address, defined in 128 byte units */
        tlan_ctx->base = txq->tx_paddr >> 7;

        tlan_ctx->pf_num = hw->pf_id;

        switch (vsi->type) {
        case ICE_VSI_PF:
                tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_PF;
                break;
        case ICE_VSI_VMDQ2:
                tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VMQ;
                break;
#ifdef PCI_IOV
        case ICE_VSI_VF:
                tlan_ctx->vmvf_type = ICE_TLAN_CTX_VMVF_TYPE_VF;
                tlan_ctx->vmvf_num = hw->func_caps.vf_base_id + vsi->vf_num;
                break;
#endif
        default:
                return (ENODEV);
        }

        tlan_ctx->src_vsi = ice_get_hw_vsi_num(hw, vsi->idx);

        /* Enable TSO */
        tlan_ctx->tso_ena = 1;
        tlan_ctx->internal_usage_flag = 1;

        tlan_ctx->tso_qnum = pf_q;

        /*
         * Stick with the older legacy Tx queue interface, instead of the new
         * advanced queue interface.
         */
        tlan_ctx->legacy_int = 1;

        /* Descriptor WB mode */
        tlan_ctx->wb_mode = 0;

        return (0);
}

/**
 * ice_cfg_vsi_for_tx - Configure the hardware for Tx
 * @vsi: the VSI to configure
 *
 * Configure the device Tx queues through firmware AdminQ commands. After
 * this, Tx queues will be ready for transmit.
 */
int
ice_cfg_vsi_for_tx(struct ice_vsi *vsi)
{
        struct ice_aqc_add_tx_qgrp *qg;
        struct ice_hw *hw = &vsi->sc->hw;
        device_t dev = vsi->sc->dev;
        int status;
        int i;
        int err = 0;
        u16 qg_size, pf_q;

        qg_size = ice_struct_size(qg, txqs, 1);
        qg = (struct ice_aqc_add_tx_qgrp *)malloc(qg_size, M_ICE, M_NOWAIT|M_ZERO);
        if (!qg)
                return (ENOMEM);

        qg->num_txqs = 1;

        for (i = 0; i < vsi->num_tx_queues; i++) {
                struct ice_tlan_ctx tlan_ctx = { 0 };
                struct ice_tx_queue *txq = &vsi->tx_queues[i];

                /* Last configured queue */
                if (txq->desc_count == 0)
                        break;

                pf_q = vsi->tx_qmap[txq->me];
                qg->txqs[0].txq_id = htole16(pf_q);

                err = ice_setup_tx_ctx(txq, &tlan_ctx, pf_q);
                if (err)
                        goto free_txqg;

                ice_set_ctx(hw, (u8 *)&tlan_ctx, qg->txqs[0].txq_ctx,
                            ice_tlan_ctx_info);

                status = ice_ena_vsi_txq(hw->port_info, vsi->idx, txq->tc,
                                         txq->q_handle, 1, qg, qg_size, NULL);
                if (status) {
                        device_printf(dev,
                                      "Failed to set LAN Tx queue %d (TC %d, handle %d) context, err %s aq_err %s\n",
                                      i, txq->tc, txq->q_handle,
                                      ice_status_str(status),
                                      ice_aq_str(hw->adminq.sq_last_status));
                        err = ENODEV;
                        goto free_txqg;
                }

                /* Keep track of the Tx queue TEID */
                if (pf_q == le16toh(qg->txqs[0].txq_id))
                        txq->q_teid = le32toh(qg->txqs[0].q_teid);
        }

free_txqg:
        free(qg, M_ICE);

        return (err);
}

/**
 * ice_setup_rx_ctx - Setup an Rx context structure for a receive queue
 * @rxq: the receive queue to program
 *
 * Setup an Rx queue context structure and program it into the hardware
 * registers. This is a necessary step for enabling the Rx queue.
 *
 * @pre the VSI associated with this queue must have initialized mbuf_sz
 */
static int
ice_setup_rx_ctx(struct ice_rx_queue *rxq)
{
        struct ice_rlan_ctx rlan_ctx = {0};
        struct ice_vsi *vsi = rxq->vsi;
        struct ice_softc *sc = vsi->sc;
        struct ice_hw *hw = &sc->hw;
        int status;
        u32 rxdid = ICE_RXDID_FLEX_NIC;
        u32 regval;
        u16 pf_q;

        pf_q = vsi->rx_qmap[rxq->me];

        /* set the receive queue base address, defined in 128 byte units */
        rlan_ctx.base = rxq->rx_paddr >> 7;

        rlan_ctx.qlen = rxq->desc_count;

        rlan_ctx.dbuf = vsi->mbuf_sz >> ICE_RLAN_CTX_DBUF_S;

        /* use 32 byte descriptors */
        rlan_ctx.dsize = 1;

        /* Strip the Ethernet CRC bytes before the packet is posted to the
         * host memory.
         */
        rlan_ctx.crcstrip = 1;

        rlan_ctx.l2tsel = 1;

        /* don't do header splitting */
        rlan_ctx.dtype = ICE_RX_DTYPE_NO_SPLIT;
        rlan_ctx.hsplit_0 = ICE_RLAN_RX_HSPLIT_0_NO_SPLIT;
        rlan_ctx.hsplit_1 = ICE_RLAN_RX_HSPLIT_1_NO_SPLIT;

        /* strip VLAN from inner headers */
        rlan_ctx.showiv = 1;

        rlan_ctx.rxmax = min(vsi->max_frame_size,
                             ICE_MAX_RX_SEGS * vsi->mbuf_sz);

        rlan_ctx.lrxqthresh = 1;

        if (vsi->type != ICE_VSI_VF) {
                regval = rd32(hw, QRXFLXP_CNTXT(pf_q));
                regval &= ~QRXFLXP_CNTXT_RXDID_IDX_M;
                regval |= (rxdid << QRXFLXP_CNTXT_RXDID_IDX_S) &
                        QRXFLXP_CNTXT_RXDID_IDX_M;

                regval &= ~QRXFLXP_CNTXT_RXDID_PRIO_M;
                regval |= (0x03 << QRXFLXP_CNTXT_RXDID_PRIO_S) &
                        QRXFLXP_CNTXT_RXDID_PRIO_M;

                wr32(hw, QRXFLXP_CNTXT(pf_q), regval);
        }

        status = ice_write_rxq_ctx(hw, &rlan_ctx, pf_q);
        if (status) {
                device_printf(sc->dev,
                              "Failed to set LAN Rx queue context, err %s aq_err %s\n",
                              ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        wr32(hw, rxq->tail, 0);

        return 0;
}

/**
 * ice_cfg_vsi_for_rx - Configure the hardware for Rx
 * @vsi: the VSI to configure
 *
 * Prepare an Rx context descriptor and configure the device to receive
 * traffic.
 *
 * @pre the VSI must have initialized mbuf_sz
 */
int
ice_cfg_vsi_for_rx(struct ice_vsi *vsi)
{
        int i, err;

        for (i = 0; i < vsi->num_rx_queues; i++) {
                MPASS(vsi->mbuf_sz > 0);
                /* Last configured queue */
                if (vsi->rx_queues[i].desc_count == 0)
                        break;

                err = ice_setup_rx_ctx(&vsi->rx_queues[i]);
                if (err)
                        return err;
        }

        return (0);
}

/**
 * ice_is_rxq_ready - Check if an Rx queue is ready
 * @hw: ice hw structure
 * @pf_q: absolute PF queue index to check
 * @reg: on successful return, contains qrx_ctrl contents
 *
 * Reads the QRX_CTRL register and verifies if the queue is in a consistent
 * state. That is, QENA_REQ matches QENA_STAT. Used to check before making
 * a request to change the queue, as well as to verify the request has
 * finished. The queue should change status within a few microseconds, so we
 * use a small delay while polling the register.
 *
 * Returns an error code if the queue does not update after a few retries.
 */
static int
ice_is_rxq_ready(struct ice_hw *hw, int pf_q, u32 *reg)
{
        u32 qrx_ctrl, qena_req, qena_stat;
        int i;

        for (i = 0; i < ICE_Q_WAIT_RETRY_LIMIT; i++) {
                qrx_ctrl = rd32(hw, QRX_CTRL(pf_q));
                qena_req = (qrx_ctrl >> QRX_CTRL_QENA_REQ_S) & 1;
                qena_stat = (qrx_ctrl >> QRX_CTRL_QENA_STAT_S) & 1;

                /* if the request and status bits equal, then the queue is
                 * fully disabled or enabled.
                 */
                if (qena_req == qena_stat) {
                        *reg = qrx_ctrl;
                        return (0);
                }

                /* wait a few microseconds before we check again */
                DELAY(10);
        }

        return (ETIMEDOUT);
}

/**
 * ice_control_rx_queue - Configure hardware to start or stop an Rx queue
 * @vsi: VSI containing queue to enable/disable
 * @qidx: Queue index in VSI space
 * @enable: true to enable queue, false to disable
 *
 * Control the Rx queue through the QRX_CTRL register, enabling or disabling
 * it. Wait for the appropriate time to ensure that the queue has actually
 * reached the expected state.
 */
int
ice_control_rx_queue(struct ice_vsi *vsi, u16 qidx, bool enable)
{
        struct ice_hw *hw = &vsi->sc->hw;
        device_t dev = vsi->sc->dev;
        u32 qrx_ctrl = 0;
        int err;

        struct ice_rx_queue *rxq = &vsi->rx_queues[qidx];
        int pf_q = vsi->rx_qmap[rxq->me];

        err = ice_is_rxq_ready(hw, pf_q, &qrx_ctrl);
        if (err) {
                device_printf(dev,
                              "Rx queue %d is not ready\n",
                              pf_q);
                return err;
        }

        /* Skip if the queue is already in correct state */
        if (enable == !!(qrx_ctrl & QRX_CTRL_QENA_STAT_M))
                return (0);

        if (enable)
                qrx_ctrl |= QRX_CTRL_QENA_REQ_M;
        else
                qrx_ctrl &= ~QRX_CTRL_QENA_REQ_M;
        wr32(hw, QRX_CTRL(pf_q), qrx_ctrl);

        /* wait for the queue to finalize the request */
        err = ice_is_rxq_ready(hw, pf_q, &qrx_ctrl);
        if (err) {
                device_printf(dev,
                              "Rx queue %d %sable timeout\n",
                              pf_q, (enable ? "en" : "dis"));
                return err;
        }

        /* this should never happen */
        if (enable != !!(qrx_ctrl & QRX_CTRL_QENA_STAT_M)) {
                device_printf(dev,
                              "Rx queue %d invalid state\n",
                              pf_q);
                return (EDOOFUS);
        }

        return (0);
}

/**
 * ice_control_all_rx_queues - Configure hardware to start or stop the Rx queues
 * @vsi: VSI to enable/disable queues
 * @enable: true to enable queues, false to disable
 *
 * Control the Rx queues through the QRX_CTRL register, enabling or disabling
 * them. Wait for the appropriate time to ensure that the queues have actually
 * reached the expected state.
 */
int
ice_control_all_rx_queues(struct ice_vsi *vsi, bool enable)
{
        int i, err;

        /* TODO: amortize waits by changing all queues up front and then
         * checking their status afterwards. This will become more necessary
         * when we have a large number of queues.
         */
        for (i = 0; i < vsi->num_rx_queues; i++) {
                err = ice_control_rx_queue(vsi, i, enable);
                if (err)
                        break;
        }

        return (0);
}

/**
 * ice_add_mac_to_list - Add MAC filter to a MAC filter list
 * @vsi: the VSI to forward to
 * @list: list which contains MAC filter entries
 * @addr: the MAC address to be added
 * @action: filter action to perform on match
 *
 * Adds a MAC address filter to the list which will be forwarded to firmware
 * to add a series of MAC address filters.
 *
 * Returns 0 on success, and an error code on failure.
 *
 */
static int
ice_add_mac_to_list(struct ice_vsi *vsi, struct ice_list_head *list,
                    const u8 *addr, enum ice_sw_fwd_act_type action)
{
        struct ice_fltr_list_entry *entry;

        entry = (__typeof(entry))malloc(sizeof(*entry), M_ICE, M_NOWAIT|M_ZERO);
        if (!entry)
                return (ENOMEM);

        entry->fltr_info.flag = ICE_FLTR_TX;
        entry->fltr_info.src_id = ICE_SRC_ID_VSI;
        entry->fltr_info.lkup_type = ICE_SW_LKUP_MAC;
        entry->fltr_info.fltr_act = action;
        entry->fltr_info.vsi_handle = vsi->idx;
        bcopy(addr, entry->fltr_info.l_data.mac.mac_addr, ETHER_ADDR_LEN);

        LIST_ADD(&entry->list_entry, list);

        return 0;
}

/**
 * ice_free_fltr_list - Free memory associated with a MAC address list
 * @list: the list to free
 *
 * Free the memory of each entry associated with the list.
 */
static void
ice_free_fltr_list(struct ice_list_head *list)
{
        struct ice_fltr_list_entry *e, *tmp;

        LIST_FOR_EACH_ENTRY_SAFE(e, tmp, list, ice_fltr_list_entry, list_entry) {
                LIST_DEL(&e->list_entry);
                free(e, M_ICE);
        }
}

/**
 * ice_add_vsi_mac_filter - Add a MAC address filter for a VSI
 * @vsi: the VSI to add the filter for
 * @addr: MAC address to add a filter for
 *
 * Add a MAC address filter for a given VSI. This is a wrapper around
 * ice_add_mac to simplify the interface. First, it only accepts a single
 * address, so we don't have to mess around with the list setup in other
 * functions. Second, it ignores the ICE_ERR_ALREADY_EXISTS error, so that
 * callers don't need to worry about attempting to add the same filter twice.
 */
int
ice_add_vsi_mac_filter(struct ice_vsi *vsi, const u8 *addr)
{
        struct ice_list_head mac_addr_list;
        struct ice_hw *hw = &vsi->sc->hw;
        device_t dev = vsi->sc->dev;
        int status;
        int err = 0;

        INIT_LIST_HEAD(&mac_addr_list);

        err = ice_add_mac_to_list(vsi, &mac_addr_list, addr, ICE_FWD_TO_VSI);
        if (err)
                goto free_mac_list;

        status = ice_add_mac(hw, &mac_addr_list);
        if (status == ICE_ERR_ALREADY_EXISTS) {
                ; /* Don't complain if we try to add a filter that already exists */
        } else if (status) {
                device_printf(dev,
                              "Failed to add a filter for MAC %6D, err %s aq_err %s\n",
                              addr, ":",
                              ice_status_str(status),
                              ice_aq_str(hw->adminq.sq_last_status));
                err = (EIO);
        }

free_mac_list:
        ice_free_fltr_list(&mac_addr_list);
        return err;
}

/**
 * ice_cfg_pf_default_mac_filters - Setup default unicast and broadcast addrs
 * @sc: device softc structure
 *
 * Program the default unicast and broadcast filters for the PF VSI.
 */
int
ice_cfg_pf_default_mac_filters(struct ice_softc *sc)
{
        struct ice_vsi *vsi = &sc->pf_vsi;
        struct ice_hw *hw = &sc->hw;
        int err;

        /* Add the LAN MAC address */
        err = ice_add_vsi_mac_filter(vsi, hw->port_info->mac.lan_addr);
        if (err)
                return err;

        /* Add the broadcast address */
        err = ice_add_vsi_mac_filter(vsi, broadcastaddr);
        if (err)
                return err;

        return (0);
}

/**
 * ice_remove_vsi_mac_filter - Remove a MAC address filter for a VSI
 * @vsi: the VSI to add the filter for
 * @addr: MAC address to remove a filter for
 *
 * Remove a MAC address filter from a given VSI. This is a wrapper around
 * ice_remove_mac to simplify the interface. First, it only accepts a single
 * address, so we don't have to mess around with the list setup in other
 * functions. Second, it ignores the ICE_ERR_DOES_NOT_EXIST error, so that
 * callers don't need to worry about attempting to remove filters which
 * haven't yet been added.
 */
int
ice_remove_vsi_mac_filter(struct ice_vsi *vsi, const u8 *addr)
{
        struct ice_list_head mac_addr_list;
        struct ice_hw *hw = &vsi->sc->hw;
        device_t dev = vsi->sc->dev;
        int status;
        int err = 0;

        INIT_LIST_HEAD(&mac_addr_list);

        err = ice_add_mac_to_list(vsi, &mac_addr_list, addr, ICE_FWD_TO_VSI);
        if (err)
                goto free_mac_list;

        status = ice_remove_mac(hw, &mac_addr_list);
        if (status == ICE_ERR_DOES_NOT_EXIST) {
                ; /* Don't complain if we try to remove a filter that doesn't exist */
        } else if (status) {
                device_printf(dev,
                              "Failed to remove a filter for MAC %6D, err %s aq_err %s\n",
                              addr, ":",
                              ice_status_str(status),
                              ice_aq_str(hw->adminq.sq_last_status));
                err = (EIO);
        }

free_mac_list:
        ice_free_fltr_list(&mac_addr_list);
        return err;
}

/**
 * ice_rm_pf_default_mac_filters - Remove default unicast and broadcast addrs
 * @sc: device softc structure
 *
 * Remove the default unicast and broadcast filters from the PF VSI.
 */
int
ice_rm_pf_default_mac_filters(struct ice_softc *sc)
{
        struct ice_vsi *vsi = &sc->pf_vsi;
        struct ice_hw *hw = &sc->hw;
        int err;

        /* Remove the LAN MAC address */
        err = ice_remove_vsi_mac_filter(vsi, hw->port_info->mac.lan_addr);
        if (err)
                return err;

        /* Remove the broadcast address */
        err = ice_remove_vsi_mac_filter(vsi, broadcastaddr);
        if (err)
                return (EIO);

        return (0);
}

/**
 * ice_check_ctrlq_errors - Check for and report controlq errors
 * @sc: device private structure
 * @qname: name of the controlq
 * @cq: the controlq to check
 *
 * Check and report controlq errors. Currently all we do is report them to the
 * kernel message log, but we might want to improve this in the future, such
 * as to keep track of statistics.
 */
static void
ice_check_ctrlq_errors(struct ice_softc *sc, const char *qname,
                       struct ice_ctl_q_info *cq)
{
        struct ice_hw *hw = &sc->hw;
        u32 val;

        /* Check for error indications. Note that all the controlqs use the
         * same register layout, so we use the PF_FW_AxQLEN defines only.
         */
        val = rd32(hw, cq->rq.len);
        if (val & (PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
                   PF_FW_ARQLEN_ARQCRIT_M)) {
                if (val & PF_FW_ARQLEN_ARQVFE_M)
                        device_printf(sc->dev,
                                "%s Receive Queue VF Error detected\n", qname);
                if (val & PF_FW_ARQLEN_ARQOVFL_M)
                        device_printf(sc->dev,
                                "%s Receive Queue Overflow Error detected\n",
                                qname);
                if (val & PF_FW_ARQLEN_ARQCRIT_M)
                        device_printf(sc->dev,
                                "%s Receive Queue Critical Error detected\n",
                                qname);
                val &= ~(PF_FW_ARQLEN_ARQVFE_M | PF_FW_ARQLEN_ARQOVFL_M |
                         PF_FW_ARQLEN_ARQCRIT_M);
                wr32(hw, cq->rq.len, val);
        }

        val = rd32(hw, cq->sq.len);
        if (val & (PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
                   PF_FW_ATQLEN_ATQCRIT_M)) {
                if (val & PF_FW_ATQLEN_ATQVFE_M)
                        device_printf(sc->dev,
                                "%s Send Queue VF Error detected\n", qname);
                if (val & PF_FW_ATQLEN_ATQOVFL_M)
                        device_printf(sc->dev,
                                "%s Send Queue Overflow Error detected\n",
                                qname);
                if (val & PF_FW_ATQLEN_ATQCRIT_M)
                        device_printf(sc->dev,
                                "%s Send Queue Critical Error detected\n",
                                qname);
                val &= ~(PF_FW_ATQLEN_ATQVFE_M | PF_FW_ATQLEN_ATQOVFL_M |
                         PF_FW_ATQLEN_ATQCRIT_M);
                wr32(hw, cq->sq.len, val);
        }
}

/**
 * ice_process_link_event - Process a link event indication from firmware
 * @sc: device softc structure
 * @e: the received event data
 *
 * Gets the current link status from hardware, and may print a message if an
 * unqualified is detected.
 */
static void
ice_process_link_event(struct ice_softc *sc,
                       struct ice_rq_event_info __invariant_only *e)
{
        struct ice_port_info *pi = sc->hw.port_info;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;

        /* Sanity check that the data length isn't too small */
        MPASS(le16toh(e->desc.datalen) >= ICE_GET_LINK_STATUS_DATALEN_V1);

        /*
         * Even though the adapter gets link status information inside the
         * event, it needs to send a Get Link Status AQ command in order
         * to re-enable link events.
         */
        pi->phy.get_link_info = true;
        ice_get_link_status(pi, &sc->link_up);

        if (pi->phy.link_info.topo_media_conflict &
           (ICE_AQ_LINK_TOPO_CONFLICT | ICE_AQ_LINK_MEDIA_CONFLICT |
            ICE_AQ_LINK_TOPO_CORRUPT))
                device_printf(dev,
                    "Possible mis-configuration of the Ethernet port detected; please use the Intel (R) Ethernet Port Configuration Tool utility to address the issue.\n");

        if ((pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) &&
            !(pi->phy.link_info.link_info & ICE_AQ_LINK_UP)) {
                if (!(pi->phy.link_info.an_info & ICE_AQ_QUALIFIED_MODULE))
                        device_printf(dev,
                            "Link is disabled on this device because an unsupported module type was detected! Refer to the Intel (R) Ethernet Adapters and Devices User Guide for a list of supported modules.\n");
                if (pi->phy.link_info.link_cfg_err & ICE_AQ_LINK_MODULE_POWER_UNSUPPORTED)
                        device_printf(dev,
                            "The module's power requirements exceed the device's power supply. Cannot start link.\n");
                if (pi->phy.link_info.link_cfg_err & ICE_AQ_LINK_INVAL_MAX_POWER_LIMIT)
                        device_printf(dev,
                            "The installed module is incompatible with the device's NVM image. Cannot start link.\n");
        }

        if (!(pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE)) {
                if (!ice_testandset_state(&sc->state, ICE_STATE_NO_MEDIA)) {
                        status = ice_aq_set_link_restart_an(pi, false, NULL);
                        if (status && hw->adminq.sq_last_status != ICE_AQ_RC_EMODE)
                                device_printf(dev,
                                    "%s: ice_aq_set_link_restart_an: status %s, aq_err %s\n",
                                    __func__, ice_status_str(status),
                                    ice_aq_str(hw->adminq.sq_last_status));
                }
        }
        /* ICE_STATE_NO_MEDIA is cleared when polling task detects media */

        /* Indicate that link status must be reported again */
        ice_clear_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED);

        /* OS link info is updated elsewhere */
}

/**
 * ice_process_ctrlq_event - Respond to a controlq event
 * @sc: device private structure
 * @qname: the name for this controlq
 * @event: the event to process
 *
 * Perform actions in response to various controlq event notifications.
 */
static void
ice_process_ctrlq_event(struct ice_softc *sc, const char *qname,
                        struct ice_rq_event_info *event)
{
        u16 opcode;

        opcode = le16toh(event->desc.opcode);

        switch (opcode) {
        case ice_aqc_opc_get_link_status:
                ice_process_link_event(sc, event);
                break;
#ifdef PCI_IOV
        case ice_mbx_opc_send_msg_to_pf:
                ice_vc_handle_vf_msg(sc, event);
                break;
#endif
        case ice_aqc_opc_fw_logs_event:
                ice_handle_fw_log_event(sc, &event->desc, event->msg_buf);
                break;
        case ice_aqc_opc_lldp_set_mib_change:
                ice_handle_mib_change_event(sc, event);
                break;
        case ice_aqc_opc_event_lan_overflow:
                ice_handle_lan_overflow_event(sc, event);
                break;
        case ice_aqc_opc_get_health_status:
                ice_handle_health_status_event(sc, event);
                break;
        default:
                device_printf(sc->dev,
                              "%s Receive Queue unhandled event 0x%04x ignored\n",
                              qname, opcode);
        }
}

/**
 * ice_process_ctrlq - helper function to process controlq rings
 * @sc: device private structure
 * @q_type: specific control queue type
 * @pending: return parameter to track remaining events
 *
 * Process controlq events for a given control queue type. Returns zero on
 * success, and an error code on failure. If successful, pending is the number
 * of remaining events left in the queue.
 */
int
ice_process_ctrlq(struct ice_softc *sc, enum ice_ctl_q q_type, u16 *pending)
{
        struct ice_rq_event_info event = { { 0 } };
        struct ice_hw *hw = &sc->hw;
        struct ice_ctl_q_info *cq;
        int status;
        const char *qname;
        int loop = 0;

        switch (q_type) {
        case ICE_CTL_Q_ADMIN:
                cq = &hw->adminq;
                qname = "Admin";
                break;
        case ICE_CTL_Q_SB:
                cq = &hw->sbq;
                qname = "Sideband";
                break;
        case ICE_CTL_Q_MAILBOX:
                cq = &hw->mailboxq;
                qname = "Mailbox";
                break;
        default:
                device_printf(sc->dev,
                              "Unknown control queue type 0x%x\n",
                              q_type);
                return 0;
        }

        ice_check_ctrlq_errors(sc, qname, cq);

        /*
         * Control queue processing happens during the admin task which may be
         * holding a non-sleepable lock, so we *must* use M_NOWAIT here.
         */
        event.buf_len = cq->rq_buf_size;
        event.msg_buf = (u8 *)malloc(event.buf_len, M_ICE, M_ZERO | M_NOWAIT);
        if (!event.msg_buf) {
                device_printf(sc->dev,
                              "Unable to allocate memory for %s Receive Queue event\n",
                              qname);
                return (ENOMEM);
        }

        do {
                status = ice_clean_rq_elem(hw, cq, &event, pending);
                if (status == ICE_ERR_AQ_NO_WORK)
                        break;
                if (status) {
                        device_printf(sc->dev,
                                      "%s Receive Queue event error %s\n",
                                      qname, ice_status_str(status));
                        free(event.msg_buf, M_ICE);
                        return (EIO);
                }
                /* XXX should we separate this handler by controlq type? */
                ice_process_ctrlq_event(sc, qname, &event);
        } while (*pending && (++loop < ICE_CTRLQ_WORK_LIMIT));

        free(event.msg_buf, M_ICE);

        return 0;
}

/**
 * pkg_ver_empty - Check if a package version is empty
 * @pkg_ver: the package version to check
 * @pkg_name: the package name to check
 *
 * Checks if the package version structure is empty. We consider a package
 * version as empty if none of the versions are non-zero and the name string
 * is null as well.
 *
 * This is used to check if the package version was initialized by the driver,
 * as we do not expect an actual DDP package file to have a zero'd version and
 * name.
 *
 * @returns true if the package version is valid, or false otherwise.
 */
static bool
pkg_ver_empty(struct ice_pkg_ver *pkg_ver, u8 *pkg_name)
{
        return (pkg_name[0] == '\0' &&
                pkg_ver->major == 0 &&
                pkg_ver->minor == 0 &&
                pkg_ver->update == 0 &&
                pkg_ver->draft == 0);
}

/**
 * pkg_ver_compatible - Check if the package version is compatible
 * @pkg_ver: the package version to check
 *
 * Compares the package version number to the driver's expected major/minor
 * version. Returns an integer indicating whether the version is older, newer,
 * or compatible with the driver.
 *
 * @returns 0 if the package version is compatible, -1 if the package version
 * is older, and 1 if the package version is newer than the driver version.
 */
static int
pkg_ver_compatible(struct ice_pkg_ver *pkg_ver)
{
        if (pkg_ver->major > ICE_PKG_SUPP_VER_MAJ)
                return (1); /* newer */
        else if ((pkg_ver->major == ICE_PKG_SUPP_VER_MAJ) &&
                 (pkg_ver->minor > ICE_PKG_SUPP_VER_MNR))
                return (1); /* newer */
        else if ((pkg_ver->major == ICE_PKG_SUPP_VER_MAJ) &&
                 (pkg_ver->minor == ICE_PKG_SUPP_VER_MNR))
                return (0); /* compatible */
        else
                return (-1); /* older */
}

/**
 * ice_os_pkg_version_str - Format OS package version info into a sbuf
 * @hw: device hw structure
 * @buf: string buffer to store name/version string
 *
 * Formats the name and version of the OS DDP package as found in the ice_ddp
 * module into a string.
 *
 * @remark This will almost always be the same as the active package, but
 * could be different in some cases. Use ice_active_pkg_version_str to get the
 * version of the active DDP package.
 */
static void
ice_os_pkg_version_str(struct ice_hw *hw, struct sbuf *buf)
{
        char name_buf[ICE_PKG_NAME_SIZE];

        /* If the OS DDP package info is empty, use "None" */
        if (pkg_ver_empty(&hw->pkg_ver, hw->pkg_name)) {
                sbuf_printf(buf, "None");
                return;
        }

        /*
         * This should already be null-terminated, but since this is a raw
         * value from an external source, strlcpy() into a new buffer to
         * make sure.
         */
        bzero(name_buf, sizeof(name_buf));
        strlcpy(name_buf, (char *)hw->pkg_name, ICE_PKG_NAME_SIZE);

        sbuf_printf(buf, "%s version %u.%u.%u.%u",
            name_buf,
            hw->pkg_ver.major,
            hw->pkg_ver.minor,
            hw->pkg_ver.update,
            hw->pkg_ver.draft);
}

/**
 * ice_active_pkg_version_str - Format active package version info into a sbuf
 * @hw: device hw structure
 * @buf: string buffer to store name/version string
 *
 * Formats the name and version of the active DDP package info into a string
 * buffer for use.
 */
static void
ice_active_pkg_version_str(struct ice_hw *hw, struct sbuf *buf)
{
        char name_buf[ICE_PKG_NAME_SIZE];

        /* If the active DDP package info is empty, use "None" */
        if (pkg_ver_empty(&hw->active_pkg_ver, hw->active_pkg_name)) {
                sbuf_printf(buf, "None");
                return;
        }

        /*
         * This should already be null-terminated, but since this is a raw
         * value from an external source, strlcpy() into a new buffer to
         * make sure.
         */
        bzero(name_buf, sizeof(name_buf));
        strlcpy(name_buf, (char *)hw->active_pkg_name, ICE_PKG_NAME_SIZE);

        sbuf_printf(buf, "%s version %u.%u.%u.%u",
            name_buf,
            hw->active_pkg_ver.major,
            hw->active_pkg_ver.minor,
            hw->active_pkg_ver.update,
            hw->active_pkg_ver.draft);

        if (hw->active_track_id != 0)
                sbuf_printf(buf, ", track id 0x%08x", hw->active_track_id);
}

/**
 * ice_nvm_version_str - Format the NVM version information into a sbuf
 * @hw: device hw structure
 * @buf: string buffer to store version string
 *
 * Formats the NVM information including firmware version, API version, NVM
 * version, the EETRACK id, and OEM specific version information into a string
 * buffer.
 */
static void
ice_nvm_version_str(struct ice_hw *hw, struct sbuf *buf)
{
        struct ice_nvm_info *nvm = &hw->flash.nvm;
        struct ice_orom_info *orom = &hw->flash.orom;
        struct ice_netlist_info *netlist = &hw->flash.netlist;

        /* Note that the netlist versions are stored in packed Binary Coded
         * Decimal format. The use of '%x' will correctly display these as
         * decimal numbers. This works because every 4 bits will be displayed
         * as a hexadecimal digit, and the BCD format will only use the values
         * 0-9.
         */
        sbuf_printf(buf,
                    "fw %u.%u.%u api %u.%u nvm %x.%02x etid %08x netlist %x.%x.%x-%x.%x.%x.%04x oem %u.%u.%u",
                    hw->fw_maj_ver, hw->fw_min_ver, hw->fw_patch,
                    hw->api_maj_ver, hw->api_min_ver,
                    nvm->major, nvm->minor, nvm->eetrack,
                    netlist->major, netlist->minor,
                    netlist->type >> 16, netlist->type & 0xFFFF,
                    netlist->rev, netlist->cust_ver, netlist->hash,
                    orom->major, orom->build, orom->patch);
}

/**
 * ice_print_nvm_version - Print the NVM info to the kernel message log
 * @sc: the device softc structure
 *
 * Format and print an NVM version string using ice_nvm_version_str().
 */
void
ice_print_nvm_version(struct ice_softc *sc)
{
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        struct sbuf *sbuf;

        sbuf = sbuf_new_auto();
        ice_nvm_version_str(hw, sbuf);
        sbuf_finish(sbuf);
        device_printf(dev, "%s\n", sbuf_data(sbuf));
        sbuf_delete(sbuf);
}

/**
 * ice_update_port_oversize - Update port oversize stats
 * @sc: device private structure
 * @rx_errors: VSI error drops
 *
 * Add ERROR_CNT from GLV_REPC VSI register and rx_oversize stats counter
 */
static void
ice_update_port_oversize(struct ice_softc *sc, u64 rx_errors)
{
        struct ice_hw_port_stats *cur_ps;
        cur_ps = &sc->stats.cur;

        sc->soft_stats.rx_roc_error = rx_errors + cur_ps->rx_oversize;
}

/**
 * ice_update_vsi_hw_stats - Update VSI-specific ethernet statistics counters
 * @vsi: the VSI to be updated
 *
 * Reads hardware stats and updates the ice_vsi_hw_stats tracking structure with
 * the updated values.
 */
void
ice_update_vsi_hw_stats(struct ice_vsi *vsi)
{
        struct ice_eth_stats *prev_es, *cur_es;
        struct ice_hw *hw = &vsi->sc->hw;
        u16 vsi_num;

        if (!ice_is_vsi_valid(hw, vsi->idx))
                return;

        vsi_num = ice_get_hw_vsi_num(hw, vsi->idx); /* HW absolute index of a VSI */
        prev_es = &vsi->hw_stats.prev;
        cur_es = &vsi->hw_stats.cur;

#define ICE_VSI_STAT40(name, location) \
        ice_stat_update40(hw, name ## L(vsi_num), \
                          vsi->hw_stats.offsets_loaded, \
                          &prev_es->location, &cur_es->location)

#define ICE_VSI_STAT32(name, location) \
        ice_stat_update32(hw, name(vsi_num), \
                          vsi->hw_stats.offsets_loaded, \
                          &prev_es->location, &cur_es->location)

        ICE_VSI_STAT40(GLV_GORC, rx_bytes);
        ICE_VSI_STAT40(GLV_UPRC, rx_unicast);
        ICE_VSI_STAT40(GLV_MPRC, rx_multicast);
        ICE_VSI_STAT40(GLV_BPRC, rx_broadcast);
        ICE_VSI_STAT32(GLV_RDPC, rx_discards);
        ICE_VSI_STAT40(GLV_GOTC, tx_bytes);
        ICE_VSI_STAT40(GLV_UPTC, tx_unicast);
        ICE_VSI_STAT40(GLV_MPTC, tx_multicast);
        ICE_VSI_STAT40(GLV_BPTC, tx_broadcast);
        ICE_VSI_STAT32(GLV_TEPC, tx_errors);

        ice_stat_update_repc(hw, vsi->idx, vsi->hw_stats.offsets_loaded,
                             cur_es);
        ice_update_port_oversize(vsi->sc, cur_es->rx_errors);
#undef ICE_VSI_STAT40
#undef ICE_VSI_STAT32

        vsi->hw_stats.offsets_loaded = true;
}

/**
 * ice_reset_vsi_stats - Reset VSI statistics counters
 * @vsi: VSI structure
 *
 * Resets the software tracking counters for the VSI statistics, and indicate
 * that the offsets haven't been loaded. This is intended to be called
 * post-reset so that VSI statistics count from zero again.
 */
void
ice_reset_vsi_stats(struct ice_vsi *vsi)
{
        /* Reset HW stats */
        memset(&vsi->hw_stats.prev, 0, sizeof(vsi->hw_stats.prev));
        memset(&vsi->hw_stats.cur, 0, sizeof(vsi->hw_stats.cur));
        vsi->hw_stats.offsets_loaded = false;
}

/**
 * ice_update_pf_stats - Update port stats counters
 * @sc: device private softc structure
 *
 * Reads hardware statistics registers and updates the software tracking
 * structure with new values.
 */
void
ice_update_pf_stats(struct ice_softc *sc)
{
        struct ice_hw_port_stats *prev_ps, *cur_ps;
        struct ice_hw *hw = &sc->hw;
        u8 lport;

        MPASS(hw->port_info);

        prev_ps = &sc->stats.prev;
        cur_ps = &sc->stats.cur;
        lport = hw->port_info->lport;

#define ICE_PF_STAT_PFC(name, location, index) \
        ice_stat_update40(hw, name(lport, index), \
                          sc->stats.offsets_loaded, \
                          &prev_ps->location[index], &cur_ps->location[index])

#define ICE_PF_STAT40(name, location) \
        ice_stat_update40(hw, name ## L(lport), \
                          sc->stats.offsets_loaded, \
                          &prev_ps->location, &cur_ps->location)

#define ICE_PF_STAT32(name, location) \
        ice_stat_update32(hw, name(lport), \
                          sc->stats.offsets_loaded, \
                          &prev_ps->location, &cur_ps->location)

        ICE_PF_STAT40(GLPRT_GORC, eth.rx_bytes);
        ICE_PF_STAT40(GLPRT_UPRC, eth.rx_unicast);
        ICE_PF_STAT40(GLPRT_MPRC, eth.rx_multicast);
        ICE_PF_STAT40(GLPRT_BPRC, eth.rx_broadcast);
        ICE_PF_STAT40(GLPRT_GOTC, eth.tx_bytes);
        ICE_PF_STAT40(GLPRT_UPTC, eth.tx_unicast);
        ICE_PF_STAT40(GLPRT_MPTC, eth.tx_multicast);
        ICE_PF_STAT40(GLPRT_BPTC, eth.tx_broadcast);
        /* This stat register doesn't have an lport */
        ice_stat_update32(hw, PRTRPB_RDPC,
                          sc->stats.offsets_loaded,
                          &prev_ps->eth.rx_discards, &cur_ps->eth.rx_discards);

        ICE_PF_STAT32(GLPRT_TDOLD, tx_dropped_link_down);
        ICE_PF_STAT40(GLPRT_PRC64, rx_size_64);
        ICE_PF_STAT40(GLPRT_PRC127, rx_size_127);
        ICE_PF_STAT40(GLPRT_PRC255, rx_size_255);
        ICE_PF_STAT40(GLPRT_PRC511, rx_size_511);
        ICE_PF_STAT40(GLPRT_PRC1023, rx_size_1023);
        ICE_PF_STAT40(GLPRT_PRC1522, rx_size_1522);
        ICE_PF_STAT40(GLPRT_PRC9522, rx_size_big);
        ICE_PF_STAT40(GLPRT_PTC64, tx_size_64);
        ICE_PF_STAT40(GLPRT_PTC127, tx_size_127);
        ICE_PF_STAT40(GLPRT_PTC255, tx_size_255);
        ICE_PF_STAT40(GLPRT_PTC511, tx_size_511);
        ICE_PF_STAT40(GLPRT_PTC1023, tx_size_1023);
        ICE_PF_STAT40(GLPRT_PTC1522, tx_size_1522);
        ICE_PF_STAT40(GLPRT_PTC9522, tx_size_big);

        /* Update Priority Flow Control Stats */
        for (int i = 0; i <= GLPRT_PXOFFRXC_MAX_INDEX; i++) {
                ICE_PF_STAT_PFC(GLPRT_PXONRXC, priority_xon_rx, i);
                ICE_PF_STAT_PFC(GLPRT_PXOFFRXC, priority_xoff_rx, i);
                ICE_PF_STAT_PFC(GLPRT_PXONTXC, priority_xon_tx, i);
                ICE_PF_STAT_PFC(GLPRT_PXOFFTXC, priority_xoff_tx, i);
                ICE_PF_STAT_PFC(GLPRT_RXON2OFFCNT, priority_xon_2_xoff, i);
        }

        ICE_PF_STAT32(GLPRT_LXONRXC, link_xon_rx);
        ICE_PF_STAT32(GLPRT_LXOFFRXC, link_xoff_rx);
        ICE_PF_STAT32(GLPRT_LXONTXC, link_xon_tx);
        ICE_PF_STAT32(GLPRT_LXOFFTXC, link_xoff_tx);
        ICE_PF_STAT32(GLPRT_CRCERRS, crc_errors);
        ICE_PF_STAT32(GLPRT_ILLERRC, illegal_bytes);
        ICE_PF_STAT32(GLPRT_MLFC, mac_local_faults);
        ICE_PF_STAT32(GLPRT_MRFC, mac_remote_faults);
        ICE_PF_STAT32(GLPRT_RLEC, rx_len_errors);
        ICE_PF_STAT32(GLPRT_RUC, rx_undersize);
        ICE_PF_STAT32(GLPRT_RFC, rx_fragments);
        ICE_PF_STAT32(GLPRT_ROC, rx_oversize);
        ICE_PF_STAT32(GLPRT_RJC, rx_jabber);

#undef ICE_PF_STAT40
#undef ICE_PF_STAT32
#undef ICE_PF_STAT_PFC

        sc->stats.offsets_loaded = true;
}

/**
 * ice_reset_pf_stats - Reset port stats counters
 * @sc: Device private softc structure
 *
 * Reset software tracking values for statistics to zero, and indicate that
 * offsets haven't been loaded. Intended to be called after a device reset so
 * that statistics count from zero again.
 */
void
ice_reset_pf_stats(struct ice_softc *sc)
{
        memset(&sc->stats.prev, 0, sizeof(sc->stats.prev));
        memset(&sc->stats.cur, 0, sizeof(sc->stats.cur));
        sc->stats.offsets_loaded = false;
}

/**
 * ice_sysctl_show_fw - sysctl callback to show firmware information
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Callback for the fw_version sysctl, to display the current firmware
 * information found at hardware init time.
 */
static int
ice_sysctl_show_fw(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_hw *hw = &sc->hw;
        struct sbuf *sbuf;

        UNREFERENCED_PARAMETER(oidp);
        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
        ice_nvm_version_str(hw, sbuf);
        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        return (0);
}

/**
 * ice_sysctl_pba_number - sysctl callback to show PBA number
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Callback for the pba_number sysctl, used to read the Product Board Assembly
 * number for this device.
 */
static int
ice_sysctl_pba_number(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        u8 pba_string[32] = "";
        int status;

        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        status = ice_read_pba_string(hw, pba_string, sizeof(pba_string));
        if (status) {
                device_printf(dev,
                    "%s: failed to read PBA string from NVM; status %s, aq_err %s\n",
                    __func__, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        return sysctl_handle_string(oidp, pba_string, sizeof(pba_string), req);
}

/**
 * ice_sysctl_pkg_version - sysctl to show the active package version info
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Callback for the pkg_version sysctl, to display the active DDP package name
 * and version information.
 */
static int
ice_sysctl_pkg_version(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_hw *hw = &sc->hw;
        struct sbuf *sbuf;

        UNREFERENCED_PARAMETER(oidp);
        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
        ice_active_pkg_version_str(hw, sbuf);
        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        return (0);
}

/**
 * ice_sysctl_os_pkg_version - sysctl to show the OS package version info
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Callback for the pkg_version sysctl, to display the OS DDP package name and
 * version info found in the ice_ddp module.
 */
static int
ice_sysctl_os_pkg_version(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_hw *hw = &sc->hw;
        struct sbuf *sbuf;

        UNREFERENCED_PARAMETER(oidp);
        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
        ice_os_pkg_version_str(hw, sbuf);
        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        return (0);
}

/**
 * ice_sysctl_current_speed - sysctl callback to show current link speed
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Callback for the current_speed sysctl, to display the string representing
 * the current link speed.
 */
static int
ice_sysctl_current_speed(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_hw *hw = &sc->hw;
        struct sbuf *sbuf;

        UNREFERENCED_PARAMETER(oidp);
        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        sbuf = sbuf_new_for_sysctl(NULL, NULL, 10, req);
        sbuf_printf(sbuf, "%s", ice_aq_speed_to_str(hw->port_info));
        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        return (0);
}

/**
 * @var phy_link_speeds
 * @brief PHY link speed conversion array
 *
 * Array of link speeds to convert ICE_PHY_TYPE_LOW and ICE_PHY_TYPE_HIGH into
 * link speeds used by the link speed sysctls.
 *
 * @remark these are based on the indices used in the BIT() macros for the
 * ICE_PHY_TYPE_LOW_* and ICE_PHY_TYPE_HIGH_* definitions.
 */
static const uint16_t phy_link_speeds[] = {
    ICE_AQ_LINK_SPEED_100MB,
    ICE_AQ_LINK_SPEED_100MB,
    ICE_AQ_LINK_SPEED_1000MB,
    ICE_AQ_LINK_SPEED_1000MB,
    ICE_AQ_LINK_SPEED_1000MB,
    ICE_AQ_LINK_SPEED_1000MB,
    ICE_AQ_LINK_SPEED_1000MB,
    ICE_AQ_LINK_SPEED_2500MB,
    ICE_AQ_LINK_SPEED_2500MB,
    ICE_AQ_LINK_SPEED_2500MB,
    ICE_AQ_LINK_SPEED_5GB,
    ICE_AQ_LINK_SPEED_5GB,
    ICE_AQ_LINK_SPEED_10GB,
    ICE_AQ_LINK_SPEED_10GB,
    ICE_AQ_LINK_SPEED_10GB,
    ICE_AQ_LINK_SPEED_10GB,
    ICE_AQ_LINK_SPEED_10GB,
    ICE_AQ_LINK_SPEED_10GB,
    ICE_AQ_LINK_SPEED_10GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_25GB,
    ICE_AQ_LINK_SPEED_40GB,
    ICE_AQ_LINK_SPEED_40GB,
    ICE_AQ_LINK_SPEED_40GB,
    ICE_AQ_LINK_SPEED_40GB,
    ICE_AQ_LINK_SPEED_40GB,
    ICE_AQ_LINK_SPEED_40GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_50GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    /* These rates are for ICE_PHY_TYPE_HIGH_* */
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_100GB,
    ICE_AQ_LINK_SPEED_200GB,
    ICE_AQ_LINK_SPEED_200GB,
    ICE_AQ_LINK_SPEED_200GB,
    ICE_AQ_LINK_SPEED_200GB,
    ICE_AQ_LINK_SPEED_200GB,
    ICE_AQ_LINK_SPEED_200GB,
    ICE_AQ_LINK_SPEED_200GB,
    ICE_AQ_LINK_SPEED_200GB,
    ICE_AQ_LINK_SPEED_200GB,
    ICE_AQ_LINK_SPEED_200GB,
};

#define ICE_SYSCTL_HELP_ADVERTISE_SPEED         \
"\nControl advertised link speed."              \
"\nFlags:"                                      \
"\n\t   0x0 - Auto"                             \
"\n\t   0x1 - 10 Mb"                            \
"\n\t   0x2 - 100 Mb"                           \
"\n\t   0x4 - 1G"                               \
"\n\t   0x8 - 2.5G"                             \
"\n\t  0x10 - 5G"                               \
"\n\t  0x20 - 10G"                              \
"\n\t  0x40 - 20G"                              \
"\n\t  0x80 - 25G"                              \
"\n\t 0x100 - 40G"                              \
"\n\t 0x200 - 50G"                              \
"\n\t 0x400 - 100G"                             \
"\n\t 0x800 - 200G"                             \
"\n\t0x8000 - Unknown"                          \
"\n\t"                                          \
"\nUse \"sysctl -x\" to view flags properly."

#define ICE_PHYS_100MB                  \
    (ICE_PHY_TYPE_LOW_100BASE_TX |      \
     ICE_PHY_TYPE_LOW_100M_SGMII)
#define ICE_PHYS_1000MB                 \
    (ICE_PHY_TYPE_LOW_1000BASE_T |      \
     ICE_PHY_TYPE_LOW_1000BASE_SX |     \
     ICE_PHY_TYPE_LOW_1000BASE_LX |     \
     ICE_PHY_TYPE_LOW_1000BASE_KX |     \
     ICE_PHY_TYPE_LOW_1G_SGMII)
#define ICE_PHYS_2500MB                 \
    (ICE_PHY_TYPE_LOW_2500BASE_T |      \
     ICE_PHY_TYPE_LOW_2500BASE_X |      \
     ICE_PHY_TYPE_LOW_2500BASE_KX)
#define ICE_PHYS_5GB                    \
    (ICE_PHY_TYPE_LOW_5GBASE_T |        \
     ICE_PHY_TYPE_LOW_5GBASE_KR)
#define ICE_PHYS_10GB                   \
    (ICE_PHY_TYPE_LOW_10GBASE_T |       \
     ICE_PHY_TYPE_LOW_10G_SFI_DA |      \
     ICE_PHY_TYPE_LOW_10GBASE_SR |      \
     ICE_PHY_TYPE_LOW_10GBASE_LR |      \
     ICE_PHY_TYPE_LOW_10GBASE_KR_CR1 |  \
     ICE_PHY_TYPE_LOW_10G_SFI_AOC_ACC | \
     ICE_PHY_TYPE_LOW_10G_SFI_C2C)
#define ICE_PHYS_25GB                   \
    (ICE_PHY_TYPE_LOW_25GBASE_T |       \
     ICE_PHY_TYPE_LOW_25GBASE_CR |      \
     ICE_PHY_TYPE_LOW_25GBASE_CR_S |    \
     ICE_PHY_TYPE_LOW_25GBASE_CR1 |     \
     ICE_PHY_TYPE_LOW_25GBASE_SR |      \
     ICE_PHY_TYPE_LOW_25GBASE_LR |      \
     ICE_PHY_TYPE_LOW_25GBASE_KR |      \
     ICE_PHY_TYPE_LOW_25GBASE_KR_S |    \
     ICE_PHY_TYPE_LOW_25GBASE_KR1 |     \
     ICE_PHY_TYPE_LOW_25G_AUI_AOC_ACC | \
     ICE_PHY_TYPE_LOW_25G_AUI_C2C)
#define ICE_PHYS_40GB                   \
    (ICE_PHY_TYPE_LOW_40GBASE_CR4 |     \
     ICE_PHY_TYPE_LOW_40GBASE_SR4 |     \
     ICE_PHY_TYPE_LOW_40GBASE_LR4 |     \
     ICE_PHY_TYPE_LOW_40GBASE_KR4 |     \
     ICE_PHY_TYPE_LOW_40G_XLAUI_AOC_ACC | \
     ICE_PHY_TYPE_LOW_40G_XLAUI)
#define ICE_PHYS_50GB                   \
    (ICE_PHY_TYPE_LOW_50GBASE_CR2 |     \
     ICE_PHY_TYPE_LOW_50GBASE_SR2 |     \
     ICE_PHY_TYPE_LOW_50GBASE_LR2 |     \
     ICE_PHY_TYPE_LOW_50GBASE_KR2 |     \
     ICE_PHY_TYPE_LOW_50G_LAUI2_AOC_ACC | \
     ICE_PHY_TYPE_LOW_50G_LAUI2 |       \
     ICE_PHY_TYPE_LOW_50G_AUI2_AOC_ACC | \
     ICE_PHY_TYPE_LOW_50G_AUI2 |        \
     ICE_PHY_TYPE_LOW_50GBASE_CP |      \
     ICE_PHY_TYPE_LOW_50GBASE_SR |      \
     ICE_PHY_TYPE_LOW_50GBASE_FR |      \
     ICE_PHY_TYPE_LOW_50GBASE_LR |      \
     ICE_PHY_TYPE_LOW_50GBASE_KR_PAM4 | \
     ICE_PHY_TYPE_LOW_50G_AUI1_AOC_ACC | \
     ICE_PHY_TYPE_LOW_50G_AUI1)
#define ICE_PHYS_100GB_LOW              \
    (ICE_PHY_TYPE_LOW_100GBASE_CR4 |    \
     ICE_PHY_TYPE_LOW_100GBASE_SR4 |    \
     ICE_PHY_TYPE_LOW_100GBASE_LR4 |    \
     ICE_PHY_TYPE_LOW_100GBASE_KR4 |    \
     ICE_PHY_TYPE_LOW_100G_CAUI4_AOC_ACC | \
     ICE_PHY_TYPE_LOW_100G_CAUI4 |      \
     ICE_PHY_TYPE_LOW_100G_AUI4_AOC_ACC | \
     ICE_PHY_TYPE_LOW_100G_AUI4 |       \
     ICE_PHY_TYPE_LOW_100GBASE_CR_PAM4 | \
     ICE_PHY_TYPE_LOW_100GBASE_KR_PAM4 | \
     ICE_PHY_TYPE_LOW_100GBASE_CP2 |    \
     ICE_PHY_TYPE_LOW_100GBASE_SR2 |    \
     ICE_PHY_TYPE_LOW_100GBASE_DR)
#define ICE_PHYS_100GB_HIGH             \
    (ICE_PHY_TYPE_HIGH_100GBASE_KR2_PAM4 | \
     ICE_PHY_TYPE_HIGH_100G_CAUI2_AOC_ACC | \
     ICE_PHY_TYPE_HIGH_100G_CAUI2 |     \
     ICE_PHY_TYPE_HIGH_100G_AUI2_AOC_ACC | \
     ICE_PHY_TYPE_HIGH_100G_AUI2)
#define ICE_PHYS_200GB                  \
    (ICE_PHY_TYPE_HIGH_200G_CR4_PAM4 |  \
     ICE_PHY_TYPE_HIGH_200G_SR4 |       \
     ICE_PHY_TYPE_HIGH_200G_FR4 |       \
     ICE_PHY_TYPE_HIGH_200G_LR4 |       \
     ICE_PHY_TYPE_HIGH_200G_DR4 |       \
     ICE_PHY_TYPE_HIGH_200G_KR4_PAM4 |  \
     ICE_PHY_TYPE_HIGH_200G_AUI4_AOC_ACC | \
     ICE_PHY_TYPE_HIGH_200G_AUI4 |      \
     ICE_PHY_TYPE_HIGH_200G_AUI8_AOC_ACC | \
     ICE_PHY_TYPE_HIGH_200G_AUI8)

/**
 * ice_aq_phy_types_to_link_speeds - Convert the PHY Types to speeds
 * @phy_type_low: lower 64-bit PHY Type bitmask
 * @phy_type_high: upper 64-bit PHY Type bitmask
 *
 * Convert the PHY Type fields from Get PHY Abilities and Set PHY Config into
 * link speed flags. If phy_type_high has an unknown PHY type, then the return
 * value will include the "ICE_AQ_LINK_SPEED_UNKNOWN" flag as well.
 */
static u16
ice_aq_phy_types_to_link_speeds(u64 phy_type_low, u64 phy_type_high)
{
        u16 sysctl_speeds = 0;
        int bit;

        /* coverity[address_of] */
        for_each_set_bit(bit, &phy_type_low, 64)
                sysctl_speeds |= phy_link_speeds[bit];

        /* coverity[address_of] */
        for_each_set_bit(bit, &phy_type_high, 64) {
                if ((bit + 64) < (int)ARRAY_SIZE(phy_link_speeds))
                        sysctl_speeds |= phy_link_speeds[bit + 64];
                else
                        sysctl_speeds |= ICE_AQ_LINK_SPEED_UNKNOWN;
        }

        return (sysctl_speeds);
}

/**
 * ice_sysctl_speeds_to_aq_phy_types - Convert sysctl speed flags to AQ PHY flags
 * @sysctl_speeds: 16-bit sysctl speeds or AQ_LINK_SPEED flags
 * @phy_type_low: output parameter for lower AQ PHY flags
 * @phy_type_high: output parameter for higher AQ PHY flags
 *
 * Converts the given link speed flags into AQ PHY type flag sets appropriate
 * for use in a Set PHY Config command.
 */
static void
ice_sysctl_speeds_to_aq_phy_types(u16 sysctl_speeds, u64 *phy_type_low,
                                  u64 *phy_type_high)
{
        *phy_type_low = 0, *phy_type_high = 0;

        if (sysctl_speeds & ICE_AQ_LINK_SPEED_100MB)
                *phy_type_low |= ICE_PHYS_100MB;
        if (sysctl_speeds & ICE_AQ_LINK_SPEED_1000MB)
                *phy_type_low |= ICE_PHYS_1000MB;
        if (sysctl_speeds & ICE_AQ_LINK_SPEED_2500MB)
                *phy_type_low |= ICE_PHYS_2500MB;
        if (sysctl_speeds & ICE_AQ_LINK_SPEED_5GB)
                *phy_type_low |= ICE_PHYS_5GB;
        if (sysctl_speeds & ICE_AQ_LINK_SPEED_10GB)
                *phy_type_low |= ICE_PHYS_10GB;
        if (sysctl_speeds & ICE_AQ_LINK_SPEED_25GB)
                *phy_type_low |= ICE_PHYS_25GB;
        if (sysctl_speeds & ICE_AQ_LINK_SPEED_40GB)
                *phy_type_low |= ICE_PHYS_40GB;
        if (sysctl_speeds & ICE_AQ_LINK_SPEED_50GB)
                *phy_type_low |= ICE_PHYS_50GB;
        if (sysctl_speeds & ICE_AQ_LINK_SPEED_100GB) {
                *phy_type_low |= ICE_PHYS_100GB_LOW;
                *phy_type_high |= ICE_PHYS_100GB_HIGH;
        }
        if (sysctl_speeds & ICE_AQ_LINK_SPEED_200GB)
                *phy_type_high |= ICE_PHYS_200GB;
}

/**
 * @struct ice_phy_data
 * @brief PHY caps and link speeds
 *
 * Buffer providing report mode and user speeds;
 * returning intersection of PHY types and speeds.
 */
struct ice_phy_data {
        u64 phy_low_orig;     /* PHY low quad from report */
        u64 phy_high_orig;    /* PHY high quad from report */
        u64 phy_low_intr;     /* PHY low quad intersection with user speeds */
        u64 phy_high_intr;    /* PHY high quad intersection with user speeds */
        u16 user_speeds_orig; /* Input from caller - See ICE_AQ_LINK_SPEED_* */
        u16 user_speeds_intr; /* Intersect with report speeds */
        u8 report_mode;       /* See ICE_AQC_REPORT_* */
};

/**
 * ice_intersect_phy_types_and_speeds - Return intersection of link speeds
 * @sc: device private structure
 * @phy_data: device PHY data
 *
 * On read: Displays the currently supported speeds
 * On write: Sets the device's supported speeds
 * Valid input flags: see ICE_SYSCTL_HELP_ADVERTISE_SPEED
 */
static int
ice_intersect_phy_types_and_speeds(struct ice_softc *sc,
                                   struct ice_phy_data *phy_data)
{
        struct ice_aqc_get_phy_caps_data pcaps = { 0 };
        const char *report_types[5] = { "w/o MEDIA",
                                        "w/MEDIA",
                                        "ACTIVE",
                                        "EDOOFUS", /* Not used */
                                        "DFLT" };
        struct ice_hw *hw = &sc->hw;
        struct ice_port_info *pi = hw->port_info;
        int status;
        u16 report_speeds, temp_speeds;
        u8 report_type;
        bool apply_speed_filter = false;

        switch (phy_data->report_mode) {
        case ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA:
        case ICE_AQC_REPORT_TOPO_CAP_MEDIA:
        case ICE_AQC_REPORT_ACTIVE_CFG:
        case ICE_AQC_REPORT_DFLT_CFG:
                report_type = phy_data->report_mode >> 1;
                break;
        default:
                device_printf(sc->dev,
                    "%s: phy_data.report_mode \"%u\" doesn't exist\n",
                    __func__, phy_data->report_mode);
                return (EINVAL);
        }

        /* 0 is treated as "Auto"; the driver will handle selecting the
         * correct speeds. Including, in some cases, applying an override
         * if provided.
         */
        if (phy_data->user_speeds_orig == 0)
                phy_data->user_speeds_orig = USHRT_MAX;
        else if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_LENIENT_LINK_MODE))
                apply_speed_filter = true;

        status = ice_aq_get_phy_caps(pi, false, phy_data->report_mode, &pcaps, NULL);
        if (status) {
                device_printf(sc->dev,
                    "%s: ice_aq_get_phy_caps (%s) failed; status %s, aq_err %s\n",
                    __func__, report_types[report_type],
                    ice_status_str(status),
                    ice_aq_str(sc->hw.adminq.sq_last_status));
                return (EIO);
        }

        phy_data->phy_low_orig = le64toh(pcaps.phy_type_low);
        phy_data->phy_high_orig = le64toh(pcaps.phy_type_high);
        report_speeds = ice_aq_phy_types_to_link_speeds(phy_data->phy_low_orig,
            phy_data->phy_high_orig);
        if (apply_speed_filter) {
                temp_speeds = ice_apply_supported_speed_filter(report_speeds,
                    pcaps.module_type[0]);
                if ((phy_data->user_speeds_orig & temp_speeds) == 0) {
                        device_printf(sc->dev,
                            "User-specified speeds (\"0x%04X\") not supported\n",
                            phy_data->user_speeds_orig);
                        return (EINVAL);
                }
                report_speeds = temp_speeds;
        }
        ice_sysctl_speeds_to_aq_phy_types(phy_data->user_speeds_orig,
            &phy_data->phy_low_intr, &phy_data->phy_high_intr);
        phy_data->user_speeds_intr = phy_data->user_speeds_orig & report_speeds;
        phy_data->phy_low_intr &= phy_data->phy_low_orig;
        phy_data->phy_high_intr &= phy_data->phy_high_orig;

        return (0);
 }

/**
 * ice_sysctl_advertise_speed - Display/change link speeds supported by port
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Displays the currently supported speeds
 * On write: Sets the device's supported speeds
 * Valid input flags: see ICE_SYSCTL_HELP_ADVERTISE_SPEED
 */
static int
ice_sysctl_advertise_speed(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_port_info *pi = sc->hw.port_info;
        struct ice_phy_data phy_data = { 0 };
        device_t dev = sc->dev;
        u16 sysctl_speeds;
        int ret;

        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        /* Get the current speeds from the adapter's "active" configuration. */
        phy_data.report_mode = ICE_AQC_REPORT_ACTIVE_CFG;
        ret = ice_intersect_phy_types_and_speeds(sc, &phy_data);
        if (ret) {
                /* Error message already printed within function */
                return (ret);
        }

        sysctl_speeds = phy_data.user_speeds_intr;

        ret = sysctl_handle_16(oidp, &sysctl_speeds, 0, req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        if (sysctl_speeds > ICE_SYSCTL_SPEEDS_VALID_RANGE) {
                device_printf(dev,
                              "%s: \"%u\" is outside of the range of acceptable values.\n",
                              __func__, sysctl_speeds);
                return (EINVAL);
        }

        pi->phy.curr_user_speed_req = sysctl_speeds;

        if (!ice_test_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN) &&
            !sc->link_up && !(if_getflags(sc->ifp) & IFF_UP))
                return 0;

        /* Apply settings requested by user */
        return ice_apply_saved_phy_cfg(sc, ICE_APPLY_LS);
}

#define ICE_SYSCTL_HELP_FEC_CONFIG                      \
"\nDisplay or set the port's requested FEC mode."       \
"\n\tauto - " ICE_FEC_STRING_AUTO                       \
"\n\tfc - " ICE_FEC_STRING_BASER                        \
"\n\trs - " ICE_FEC_STRING_RS                           \
"\n\tnone - " ICE_FEC_STRING_NONE                       \
"\nEither of the left or right strings above can be used to set the requested mode."

/**
 * ice_sysctl_fec_config - Display/change the configured FEC mode
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Displays the configured FEC mode
 * On write: Sets the device's FEC mode to the input string, if it's valid.
 * Valid input strings: see ICE_SYSCTL_HELP_FEC_CONFIG
 */
static int
ice_sysctl_fec_config(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_port_info *pi = sc->hw.port_info;
        enum ice_fec_mode new_mode;
        device_t dev = sc->dev;
        char req_fec[32];
        int ret;

        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        bzero(req_fec, sizeof(req_fec));
        strlcpy(req_fec, ice_requested_fec_mode(pi), sizeof(req_fec));

        ret = sysctl_handle_string(oidp, req_fec, sizeof(req_fec), req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        if (strcmp(req_fec, "auto") == 0 ||
            strcmp(req_fec, ice_fec_str(ICE_FEC_AUTO)) == 0) {
                if (sc->allow_no_fec_mod_in_auto)
                        new_mode = ICE_FEC_DIS_AUTO;
                else
                        new_mode = ICE_FEC_AUTO;
        } else if (strcmp(req_fec, "fc") == 0 ||
            strcmp(req_fec, ice_fec_str(ICE_FEC_BASER)) == 0) {
                new_mode = ICE_FEC_BASER;
        } else if (strcmp(req_fec, "rs") == 0 ||
            strcmp(req_fec, ice_fec_str(ICE_FEC_RS)) == 0) {
                new_mode = ICE_FEC_RS;
        } else if (strcmp(req_fec, "none") == 0 ||
            strcmp(req_fec, ice_fec_str(ICE_FEC_NONE)) == 0) {
                new_mode = ICE_FEC_NONE;
        } else {
                device_printf(dev,
                    "%s: \"%s\" is not a valid FEC mode\n",
                    __func__, req_fec);
                return (EINVAL);
        }

        /* Cache user FEC mode for later link ups */
        pi->phy.curr_user_fec_req = new_mode;

        if (!ice_test_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN) && !sc->link_up)
                return 0;

        /* Apply settings requested by user */
        return ice_apply_saved_phy_cfg(sc, ICE_APPLY_FEC);
}

/**
 * ice_sysctl_negotiated_fec - Display the negotiated FEC mode on the link
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Displays the negotiated FEC mode, in a string
 */
static int
ice_sysctl_negotiated_fec(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_hw *hw = &sc->hw;
        char neg_fec[32];
        int ret;

        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        /* Copy const string into a buffer to drop const qualifier */
        bzero(neg_fec, sizeof(neg_fec));
        strlcpy(neg_fec, ice_negotiated_fec_mode(hw->port_info), sizeof(neg_fec));

        ret = sysctl_handle_string(oidp, neg_fec, 0, req);
        if (req->newptr != NULL)
                return (EPERM);

        return (ret);
}

#define ICE_SYSCTL_HELP_FC_CONFIG                               \
"\nDisplay or set the port's advertised flow control mode.\n"   \
"\t0 - " ICE_FC_STRING_NONE                                     \
"\n\t1 - " ICE_FC_STRING_RX                                     \
"\n\t2 - " ICE_FC_STRING_TX                                     \
"\n\t3 - " ICE_FC_STRING_FULL                                   \
"\nEither the numbers or the strings above can be used to set the advertised mode."

/**
 * ice_sysctl_fc_config - Display/change the advertised flow control mode
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Displays the configured flow control mode
 * On write: Sets the device's flow control mode to the input, if it's valid.
 * Valid input strings: see ICE_SYSCTL_HELP_FC_CONFIG
 */
static int
ice_sysctl_fc_config(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_port_info *pi = sc->hw.port_info;
        struct ice_aqc_get_phy_caps_data pcaps = { 0 };
        enum ice_fc_mode old_mode, new_mode;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;
        int ret, fc_num;
        bool mode_set = false;
        struct sbuf buf;
        char *fc_str_end;
        char fc_str[32];

        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG,
                                     &pcaps, NULL);
        if (status) {
                device_printf(dev,
                    "%s: ice_aq_get_phy_caps failed; status %s, aq_err %s\n",
                    __func__, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        /* Convert HW response format to SW enum value */
        if ((pcaps.caps & ICE_AQC_PHY_EN_TX_LINK_PAUSE) &&
            (pcaps.caps & ICE_AQC_PHY_EN_RX_LINK_PAUSE))
                old_mode = ICE_FC_FULL;
        else if (pcaps.caps & ICE_AQC_PHY_EN_TX_LINK_PAUSE)
                old_mode = ICE_FC_TX_PAUSE;
        else if (pcaps.caps & ICE_AQC_PHY_EN_RX_LINK_PAUSE)
                old_mode = ICE_FC_RX_PAUSE;
        else
                old_mode = ICE_FC_NONE;

        /* Create "old" string for output */
        bzero(fc_str, sizeof(fc_str));
        sbuf_new_for_sysctl(&buf, fc_str, sizeof(fc_str), req);
        sbuf_printf(&buf, "%d<%s>", old_mode, ice_fc_str(old_mode));
        sbuf_finish(&buf);
        sbuf_delete(&buf);

        ret = sysctl_handle_string(oidp, fc_str, sizeof(fc_str), req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        /* Try to parse input as a string, first */
        if (strcasecmp(ice_fc_str(ICE_FC_FULL), fc_str) == 0) {
                new_mode = ICE_FC_FULL;
                mode_set = true;
        }
        else if (strcasecmp(ice_fc_str(ICE_FC_TX_PAUSE), fc_str) == 0) {
                new_mode = ICE_FC_TX_PAUSE;
                mode_set = true;
        }
        else if (strcasecmp(ice_fc_str(ICE_FC_RX_PAUSE), fc_str) == 0) {
                new_mode = ICE_FC_RX_PAUSE;
                mode_set = true;
        }
        else if (strcasecmp(ice_fc_str(ICE_FC_NONE), fc_str) == 0) {
                new_mode = ICE_FC_NONE;
                mode_set = true;
        }

        /*
         * Then check if it's an integer, for compatibility with the method
         * used in older drivers.
         */
        if (!mode_set) {
                fc_num = strtol(fc_str, &fc_str_end, 0);
                if (fc_str_end == fc_str)
                        fc_num = -1;
                switch (fc_num) {
                case 3:
                        new_mode = ICE_FC_FULL;
                        break;
                case 2:
                        new_mode = ICE_FC_TX_PAUSE;
                        break;
                case 1:
                        new_mode = ICE_FC_RX_PAUSE;
                        break;
                case 0:
                        new_mode = ICE_FC_NONE;
                        break;
                default:
                        device_printf(dev,
                            "%s: \"%s\" is not a valid flow control mode\n",
                            __func__, fc_str);
                        return (EINVAL);
                }
        }

        /* Save flow control mode from user */
        pi->phy.curr_user_fc_req = new_mode;

        /* Turn off Priority Flow Control when Link Flow Control is enabled */
        if ((hw->port_info->qos_cfg.is_sw_lldp) &&
            (hw->port_info->qos_cfg.local_dcbx_cfg.pfc.pfcena != 0) &&
            (new_mode != ICE_FC_NONE)) {
                ret = ice_config_pfc(sc, 0x0);
                if (ret)
                        return (ret);
        }

        if (!ice_test_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN) && !sc->link_up)
                return 0;

        /* Apply settings requested by user */
        return ice_apply_saved_phy_cfg(sc, ICE_APPLY_FC);
}

/**
 * ice_sysctl_negotiated_fc - Display currently negotiated FC mode
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Displays the currently negotiated flow control settings.
 *
 * If link is not established, this will report ICE_FC_NONE, as no flow
 * control is negotiated while link is down.
 */
static int
ice_sysctl_negotiated_fc(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_port_info *pi = sc->hw.port_info;
        const char *negotiated_fc;

        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        negotiated_fc = ice_flowcontrol_mode(pi);

        return sysctl_handle_string(oidp, __DECONST(char *, negotiated_fc), 0, req);
}

/**
 * __ice_sysctl_phy_type_handler - Display/change supported PHY types/speeds
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 * @is_phy_type_high: if true, handle the high PHY type instead of the low PHY type
 *
 * Private handler for phy_type_high and phy_type_low sysctls.
 */
static int
__ice_sysctl_phy_type_handler(SYSCTL_HANDLER_ARGS, bool is_phy_type_high)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_aqc_get_phy_caps_data pcaps = { 0 };
        struct ice_aqc_set_phy_cfg_data cfg = { 0 };
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;
        uint64_t types;
        int ret;

        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        status = ice_aq_get_phy_caps(hw->port_info, false, ICE_AQC_REPORT_ACTIVE_CFG,
                                     &pcaps, NULL);
        if (status) {
                device_printf(dev,
                    "%s: ice_aq_get_phy_caps failed; status %s, aq_err %s\n",
                    __func__, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        if (is_phy_type_high)
                types = pcaps.phy_type_high;
        else
                types = pcaps.phy_type_low;

        ret = sysctl_handle_64(oidp, &types, sizeof(types), req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        ice_copy_phy_caps_to_cfg(hw->port_info, &pcaps, &cfg);

        if (is_phy_type_high)
                cfg.phy_type_high = types & hw->port_info->phy.phy_type_high;
        else
                cfg.phy_type_low = types & hw->port_info->phy.phy_type_low;
        cfg.caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT;

        status = ice_aq_set_phy_cfg(hw, hw->port_info, &cfg, NULL);
        if (status) {
                device_printf(dev,
                    "%s: ice_aq_set_phy_cfg failed; status %s, aq_err %s\n",
                    __func__, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        return (0);

}

/**
 * ice_sysctl_phy_type_low - Display/change supported lower PHY types/speeds
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Displays the currently supported lower PHY types
 * On write: Sets the device's supported low PHY types
 */
static int
ice_sysctl_phy_type_low(SYSCTL_HANDLER_ARGS)
{
        return __ice_sysctl_phy_type_handler(oidp, arg1, arg2, req, false);
}

/**
 * ice_sysctl_phy_type_high - Display/change supported higher PHY types/speeds
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Displays the currently supported higher PHY types
 * On write: Sets the device's supported high PHY types
 */
static int
ice_sysctl_phy_type_high(SYSCTL_HANDLER_ARGS)
{
        return __ice_sysctl_phy_type_handler(oidp, arg1, arg2, req, true);
}

/**
 * ice_sysctl_phy_caps - Display response from Get PHY abililties
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 * @report_mode: the mode to report
 *
 * On read: Display the response from Get PHY abillities with the given report
 * mode.
 */
static int
ice_sysctl_phy_caps(SYSCTL_HANDLER_ARGS, u8 report_mode)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_aqc_get_phy_caps_data pcaps = { 0 };
        struct ice_hw *hw = &sc->hw;
        struct ice_port_info *pi = hw->port_info;
        device_t dev = sc->dev;
        int status;
        int ret;

        UNREFERENCED_PARAMETER(arg2);

        ret = priv_check(curthread, PRIV_DRIVER);
        if (ret)
                return (ret);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        status = ice_aq_get_phy_caps(pi, true, report_mode, &pcaps, NULL);
        if (status) {
                device_printf(dev,
                    "%s: ice_aq_get_phy_caps failed; status %s, aq_err %s\n",
                    __func__, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        ret = sysctl_handle_opaque(oidp, &pcaps, sizeof(pcaps), req);
        if (req->newptr != NULL)
                return (EPERM);

        return (ret);
}

/**
 * ice_sysctl_phy_sw_caps - Display response from Get PHY abililties
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Display the response from Get PHY abillities reporting the last
 * software configuration.
 */
static int
ice_sysctl_phy_sw_caps(SYSCTL_HANDLER_ARGS)
{
        return ice_sysctl_phy_caps(oidp, arg1, arg2, req,
                                   ICE_AQC_REPORT_ACTIVE_CFG);
}

/**
 * ice_sysctl_phy_nvm_caps - Display response from Get PHY abililties
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Display the response from Get PHY abillities reporting the NVM
 * configuration.
 */
static int
ice_sysctl_phy_nvm_caps(SYSCTL_HANDLER_ARGS)
{
        return ice_sysctl_phy_caps(oidp, arg1, arg2, req,
                                   ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA);
}

/**
 * ice_sysctl_phy_topo_caps - Display response from Get PHY abililties
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Display the response from Get PHY abillities reporting the
 * topology configuration.
 */
static int
ice_sysctl_phy_topo_caps(SYSCTL_HANDLER_ARGS)
{
        return ice_sysctl_phy_caps(oidp, arg1, arg2, req,
                                   ICE_AQC_REPORT_TOPO_CAP_MEDIA);
}

/**
 * ice_sysctl_phy_link_status - Display response from Get Link Status
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Display the response from firmware for the Get Link Status
 * request.
 */
static int
ice_sysctl_phy_link_status(SYSCTL_HANDLER_ARGS)
{
        struct ice_aqc_get_link_status_data link_data = { 0 };
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_hw *hw = &sc->hw;
        struct ice_port_info *pi = hw->port_info;
        struct ice_aqc_get_link_status *resp;
        struct ice_aq_desc desc;
        device_t dev = sc->dev;
        int status;
        int ret;

        UNREFERENCED_PARAMETER(arg2);

        /*
         * Ensure that only contexts with driver privilege are allowed to
         * access this information
         */
        ret = priv_check(curthread, PRIV_DRIVER);
        if (ret)
                return (ret);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        ice_fill_dflt_direct_cmd_desc(&desc, ice_aqc_opc_get_link_status);
        resp = &desc.params.get_link_status;
        resp->lport_num = pi->lport;

        status = ice_aq_send_cmd(hw, &desc, &link_data, sizeof(link_data), NULL);
        if (status) {
                device_printf(dev,
                    "%s: ice_aq_send_cmd failed; status %s, aq_err %s\n",
                    __func__, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        ret = sysctl_handle_opaque(oidp, &link_data, sizeof(link_data), req);
        if (req->newptr != NULL)
                return (EPERM);

        return (ret);
}

/**
 * ice_sysctl_fw_cur_lldp_persist_status - Display current FW LLDP status
 * @oidp: sysctl oid structure
 * @arg1: pointer to private softc structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Displays current persistent LLDP status.
 */
static int
ice_sysctl_fw_cur_lldp_persist_status(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;
        struct sbuf *sbuf;
        u32 lldp_state;

        UNREFERENCED_PARAMETER(arg2);
        UNREFERENCED_PARAMETER(oidp);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        status = ice_get_cur_lldp_persist_status(hw, &lldp_state);
        if (status) {
                device_printf(dev,
                    "Could not acquire current LLDP persistence status, err %s aq_err %s\n",
                    ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
        sbuf_printf(sbuf, "%s", ice_fw_lldp_status(lldp_state));
        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        return (0);
}

/**
 * ice_sysctl_fw_dflt_lldp_persist_status - Display default FW LLDP status
 * @oidp: sysctl oid structure
 * @arg1: pointer to private softc structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Displays default persistent LLDP status.
 */
static int
ice_sysctl_fw_dflt_lldp_persist_status(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;
        struct sbuf *sbuf;
        u32 lldp_state;

        UNREFERENCED_PARAMETER(arg2);
        UNREFERENCED_PARAMETER(oidp);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        status = ice_get_dflt_lldp_persist_status(hw, &lldp_state);
        if (status) {
                device_printf(dev,
                    "Could not acquire default LLDP persistence status, err %s aq_err %s\n",
                    ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
        sbuf_printf(sbuf, "%s", ice_fw_lldp_status(lldp_state));
        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        return (0);
}

/**
 * ice_dscp_is_mapped - Check for non-zero DSCP to TC mappings
 * @dcbcfg: Configuration struct to check for mappings in
 *
 * @return true if there exists a non-zero DSCP to TC mapping
 * inside the input DCB configuration struct.
 */
static bool
ice_dscp_is_mapped(struct ice_dcbx_cfg *dcbcfg)
{
        for (int i = 0; i < ICE_DSCP_NUM_VAL; i++)
                if (dcbcfg->dscp_map[i] != 0)
                        return (true);

        return (false);
}

#define ICE_SYSCTL_HELP_FW_LLDP_AGENT   \
"\nDisplay or change FW LLDP agent state:" \
"\n\t0 - disabled"                      \
"\n\t1 - enabled"

/**
 * ice_sysctl_fw_lldp_agent - Display or change the FW LLDP agent status
 * @oidp: sysctl oid structure
 * @arg1: pointer to private softc structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Displays whether the FW LLDP agent is running
 * On write: Persistently enables or disables the FW LLDP agent
 */
static int
ice_sysctl_fw_lldp_agent(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_dcbx_cfg *local_dcbx_cfg;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;
        int ret;
        u32 old_state;
        u8 fw_lldp_enabled;
        bool retried_start_lldp = false;

        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        status = ice_get_cur_lldp_persist_status(hw, &old_state);
        if (status) {
                device_printf(dev,
                    "Could not acquire current LLDP persistence status, err %s aq_err %s\n",
                    ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        if (old_state > ICE_LLDP_ADMINSTATUS_ENA_RXTX) {
                status = ice_get_dflt_lldp_persist_status(hw, &old_state);
                if (status) {
                        device_printf(dev,
                            "Could not acquire default LLDP persistence status, err %s aq_err %s\n",
                            ice_status_str(status),
                            ice_aq_str(hw->adminq.sq_last_status));
                        return (EIO);
                }
        }
        if (old_state == 0)
                fw_lldp_enabled = false;
        else
                fw_lldp_enabled = true;

        ret = sysctl_handle_bool(oidp, &fw_lldp_enabled, 0, req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        if (old_state == 0 && fw_lldp_enabled == false)
                return (0);

        if (old_state != 0 && fw_lldp_enabled == true)
                return (0);

        /* Block transition to FW LLDP if DSCP mode is enabled */
        local_dcbx_cfg = &hw->port_info->qos_cfg.local_dcbx_cfg;
        if ((local_dcbx_cfg->pfc_mode == ICE_QOS_MODE_DSCP) ||
            ice_dscp_is_mapped(local_dcbx_cfg)) {
                device_printf(dev,
                              "Cannot enable FW-LLDP agent while DSCP QoS is active.\n");
                return (EOPNOTSUPP);
        }

        if (fw_lldp_enabled == false) {
                status = ice_aq_stop_lldp(hw, true, true, NULL);
                /* EPERM is returned if the LLDP agent is already shutdown */
                if (status && hw->adminq.sq_last_status != ICE_AQ_RC_EPERM) {
                        device_printf(dev,
                            "%s: ice_aq_stop_lldp failed; status %s, aq_err %s\n",
                            __func__, ice_status_str(status),
                            ice_aq_str(hw->adminq.sq_last_status));
                        return (EIO);
                }
                ice_aq_set_dcb_parameters(hw, true, NULL);
                hw->port_info->qos_cfg.is_sw_lldp = true;
                ice_add_rx_lldp_filter(sc);
        } else {
                ice_del_rx_lldp_filter(sc);
retry_start_lldp:
                status = ice_aq_start_lldp(hw, true, NULL);
                if (status) {
                        switch (hw->adminq.sq_last_status) {
                        /* EEXIST is returned if the LLDP agent is already started */
                        case ICE_AQ_RC_EEXIST:
                                break;
                        case ICE_AQ_RC_EAGAIN:
                                /* Retry command after a 2 second wait */
                                if (retried_start_lldp == false) {
                                        retried_start_lldp = true;
                                        pause("slldp", ICE_START_LLDP_RETRY_WAIT);
                                        goto retry_start_lldp;
                                }
                                /* Fallthrough */
                        default:
                                device_printf(dev,
                                    "%s: ice_aq_start_lldp failed; status %s, aq_err %s\n",
                                    __func__, ice_status_str(status),
                                    ice_aq_str(hw->adminq.sq_last_status));
                                return (EIO);
                        }
                }
                ice_start_dcbx_agent(sc);

                /* Init DCB needs to be done during enabling LLDP to properly
                 * propagate the configuration.
                 */
                status = ice_init_dcb(hw, true);
                if (status) {
                        device_printf(dev,
                            "%s: ice_init_dcb failed; status %s, aq_err %s\n",
                            __func__, ice_status_str(status),
                            ice_aq_str(hw->adminq.sq_last_status));
                        hw->port_info->qos_cfg.dcbx_status = ICE_DCBX_STATUS_NOT_STARTED;
                }
        }

        return (ret);
}

#define ICE_SYSCTL_HELP_ETS_MIN_RATE \
"\nIn FW DCB mode (fw_lldp_agent=1), displays the current ETS bandwidth table." \
"\nIn SW DCB mode, displays and allows setting the table." \
"\nInput must be in the format e.g. 30,10,10,10,10,10,10,10" \
"\nWhere the bandwidth total must add up to 100"

/**
 * ice_sysctl_ets_min_rate - Report/configure ETS bandwidth
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Returns the current ETS TC bandwidth table
 * cached by the driver.
 *
 * In SW DCB mode this sysctl also accepts a value that will
 * be sent to the firmware for configuration.
 */
static int
ice_sysctl_ets_min_rate(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_dcbx_cfg *local_dcbx_cfg;
        struct ice_port_info *pi;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;
        struct sbuf *sbuf;
        int ret;

        /* Store input rates from user */
        char ets_user_buf[128] = "";
        u8 new_ets_table[ICE_MAX_TRAFFIC_CLASS] = {};

        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        if (req->oldptr == NULL && req->newptr == NULL) {
                ret = SYSCTL_OUT(req, 0, 128);
                return (ret);
        }

        pi = hw->port_info;
        local_dcbx_cfg = &pi->qos_cfg.local_dcbx_cfg;

        sbuf = sbuf_new(NULL, ets_user_buf, 128, SBUF_FIXEDLEN | SBUF_INCLUDENUL);

        /* Format ETS BW data for output */
        for (int i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++) {
                sbuf_printf(sbuf, "%d", local_dcbx_cfg->etscfg.tcbwtable[i]);
                if (i != ICE_MAX_TRAFFIC_CLASS - 1)
                        sbuf_printf(sbuf, ",");
        }

        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        /* Read in the new ETS values */
        ret = sysctl_handle_string(oidp, ets_user_buf, sizeof(ets_user_buf), req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        /* Don't allow setting changes in FW DCB mode */
        if (!hw->port_info->qos_cfg.is_sw_lldp)
                return (EPERM);

        ret = ice_ets_str_to_tbl(ets_user_buf, new_ets_table, 100);
        if (ret) {
                device_printf(dev, "%s: Could not parse input BW table: %s\n",
                    __func__, ets_user_buf);
                return (ret);
        }

        if (!ice_check_ets_bw(new_ets_table)) {
                device_printf(dev, "%s: Bandwidth sum does not equal 100: %s\n",
                    __func__, ets_user_buf);
                return (EINVAL);
        }

        memcpy(local_dcbx_cfg->etscfg.tcbwtable, new_ets_table,
            sizeof(new_ets_table));

        /* If BW > 0, then set TSA entry to 2 */
        for (int i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++) {
                if (new_ets_table[i] > 0)
                        local_dcbx_cfg->etscfg.tsatable[i] = 2;
                else
                        local_dcbx_cfg->etscfg.tsatable[i] = 0;
        }
        local_dcbx_cfg->etscfg.willing = 0;
        local_dcbx_cfg->etsrec = local_dcbx_cfg->etscfg;
        local_dcbx_cfg->app_mode = ICE_DCBX_APPS_NON_WILLING;

        status = ice_set_dcb_cfg(pi);
        if (status) {
                device_printf(dev,
                    "%s: Failed to set DCB config; status %s, aq_err %s\n",
                    __func__, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        ice_do_dcb_reconfig(sc, false);

        return (0);
}

#define ICE_SYSCTL_HELP_UP2TC_MAP \
"\nIn FW DCB mode (fw_lldp_agent=1), displays the current ETS priority assignment table." \
"\nIn SW DCB mode, displays and allows setting the table." \
"\nInput must be in this format: 0,1,2,3,4,5,6,7" \
"\nWhere the 1st number is the TC for UP0, 2nd number is the TC for UP1, etc"

/**
 * ice_sysctl_up2tc_map - Report or configure UP2TC mapping
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * In FW DCB mode, returns the current ETS prio table /
 * UP2TC mapping from the local MIB.
 *
 * In SW DCB mode this sysctl also accepts a value that will
 * be sent to the firmware for configuration.
 */
static int
ice_sysctl_up2tc_map(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_dcbx_cfg *local_dcbx_cfg;
        struct ice_port_info *pi;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;
        struct sbuf *sbuf;
        int ret;

        /* Store input rates from user */
        char up2tc_user_buf[128] = "";
        /* This array is indexed by UP, not TC */
        u8 new_up2tc[ICE_MAX_TRAFFIC_CLASS] = {};

        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        if (req->oldptr == NULL && req->newptr == NULL) {
                ret = SYSCTL_OUT(req, 0, 128);
                return (ret);
        }

        pi = hw->port_info;
        local_dcbx_cfg = &pi->qos_cfg.local_dcbx_cfg;

        sbuf = sbuf_new(NULL, up2tc_user_buf, 128, SBUF_FIXEDLEN | SBUF_INCLUDENUL);

        /* Format ETS Priority Mapping Table for output */
        for (int i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++) {
                sbuf_printf(sbuf, "%d", local_dcbx_cfg->etscfg.prio_table[i]);
                if (i != ICE_MAX_TRAFFIC_CLASS - 1)
                        sbuf_printf(sbuf, ",");
        }

        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        /* Read in the new ETS priority mapping */
        ret = sysctl_handle_string(oidp, up2tc_user_buf, sizeof(up2tc_user_buf), req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        /* Don't allow setting changes in FW DCB mode */
        if (!hw->port_info->qos_cfg.is_sw_lldp)
                return (EPERM);

        ret = ice_ets_str_to_tbl(up2tc_user_buf, new_up2tc,
            ICE_MAX_TRAFFIC_CLASS - 1);
        if (ret) {
                device_printf(dev, "%s: Could not parse input priority assignment table: %s\n",
                    __func__, up2tc_user_buf);
                return (ret);
        }

        /* Prepare updated ETS CFG/REC TLVs */
        memcpy(local_dcbx_cfg->etscfg.prio_table, new_up2tc,
            sizeof(new_up2tc));
        memcpy(local_dcbx_cfg->etsrec.prio_table, new_up2tc,
            sizeof(new_up2tc));

        status = ice_set_dcb_cfg(pi);
        if (status) {
                device_printf(dev,
                    "%s: Failed to set DCB config; status %s, aq_err %s\n",
                    __func__, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        ice_do_dcb_reconfig(sc, false);

        return (0);
}

/**
 * ice_config_pfc - helper function to set PFC config in FW
 * @sc: device private structure
 * @new_mode: bit flags indicating PFC status for TCs
 *
 * @pre must be in SW DCB mode
 *
 * Configures the driver's local PFC TLV and sends it to the
 * FW for configuration, then reconfigures the driver/VSI
 * for DCB if needed.
 */
static int
ice_config_pfc(struct ice_softc *sc, u8 new_mode)
{
        struct ice_dcbx_cfg *local_dcbx_cfg;
        struct ice_hw *hw = &sc->hw;
        struct ice_port_info *pi;
        device_t dev = sc->dev;
        int status;

        pi = hw->port_info;
        local_dcbx_cfg = &pi->qos_cfg.local_dcbx_cfg;

        /* Prepare updated PFC TLV */
        local_dcbx_cfg->pfc.pfcena = new_mode;
        local_dcbx_cfg->pfc.pfccap = ICE_MAX_TRAFFIC_CLASS;
        local_dcbx_cfg->pfc.willing = 0;
        local_dcbx_cfg->pfc.mbc = 0;

        /* Warn if PFC is being disabled with RoCE v2 in use */
        if (new_mode == 0 && sc->rdma_entry.attached)
                device_printf(dev,
                    "WARNING: Recommended that Priority Flow Control is enabled when RoCEv2 is in use\n");

        status = ice_set_dcb_cfg(pi);
        if (status) {
                device_printf(dev,
                    "%s: Failed to set DCB config; status %s, aq_err %s\n",
                    __func__, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        ice_do_dcb_reconfig(sc, false);

        return (0);
}

#define ICE_SYSCTL_HELP_PFC_CONFIG \
"\nIn FW DCB mode (fw_lldp_agent=1), displays the current Priority Flow Control configuration" \
"\nIn SW DCB mode, displays and allows setting the configuration" \
"\nInput/Output is in this format: 0xff" \
"\nWhere bit position # enables/disables PFC for that Traffic Class #"

/**
 * ice_sysctl_pfc_config - Report or configure enabled PFC TCs
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * In FW DCB mode, returns a bitmap containing the current TCs
 * that have PFC enabled on them.
 *
 * In SW DCB mode this sysctl also accepts a value that will
 * be sent to the firmware for configuration.
 */
static int
ice_sysctl_pfc_config(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_dcbx_cfg *local_dcbx_cfg;
        struct ice_port_info *pi;
        struct ice_hw *hw = &sc->hw;
        int ret;

        /* Store input flags from user */
        u8 user_pfc;

        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        if (req->oldptr == NULL && req->newptr == NULL) {
                ret = SYSCTL_OUT(req, 0, sizeof(u8));
                return (ret);
        }

        pi = hw->port_info;
        local_dcbx_cfg = &pi->qos_cfg.local_dcbx_cfg;

        /* Format current PFC enable setting for output */
        user_pfc = local_dcbx_cfg->pfc.pfcena;

        /* Read in the new PFC config */
        ret = sysctl_handle_8(oidp, &user_pfc, 0, req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        /* Don't allow setting changes in FW DCB mode */
        if (!hw->port_info->qos_cfg.is_sw_lldp)
                return (EPERM);

        /* If LFC is active and PFC is going to be turned on, turn LFC off */
        if (user_pfc != 0 && pi->phy.curr_user_fc_req != ICE_FC_NONE) {
                pi->phy.curr_user_fc_req = ICE_FC_NONE;
                if (ice_test_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN) ||
                         sc->link_up) {
                        ret = ice_apply_saved_phy_cfg(sc, ICE_APPLY_FC);
                        if (ret)
                                return (ret);
                }
        }

        return ice_config_pfc(sc, user_pfc);
}

#define ICE_SYSCTL_HELP_PFC_MODE \
"\nDisplay and set the current QoS mode for the firmware" \
"\n\t0: VLAN UP mode" \
"\n\t1: DSCP mode"

/**
 * ice_sysctl_pfc_mode
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Gets and sets whether the port is in DSCP or VLAN PCP-based
 * PFC mode. This is also used to set whether DSCP or VLAN PCP
 * -based settings are configured for DCB.
 */
static int
ice_sysctl_pfc_mode(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_dcbx_cfg *local_dcbx_cfg;
        struct ice_port_info *pi;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;
        u8 user_pfc_mode, aq_pfc_mode;
        int ret;

        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        if (req->oldptr == NULL && req->newptr == NULL) {
                ret = SYSCTL_OUT(req, 0, sizeof(u8));
                return (ret);
        }

        pi = hw->port_info;
        local_dcbx_cfg = &pi->qos_cfg.local_dcbx_cfg;

        user_pfc_mode = local_dcbx_cfg->pfc_mode;

        /* Read in the new mode */
        ret = sysctl_handle_8(oidp, &user_pfc_mode, 0, req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        /* Don't allow setting changes in FW DCB mode */
        if (!hw->port_info->qos_cfg.is_sw_lldp)
                return (EPERM);

        /* Currently, there are only two modes */
        switch (user_pfc_mode) {
        case 0:
                aq_pfc_mode = ICE_AQC_PFC_VLAN_BASED_PFC;
                break;
        case 1:
                aq_pfc_mode = ICE_AQC_PFC_DSCP_BASED_PFC;
                break;
        default:
                device_printf(dev,
                    "%s: Valid input range is 0-1 (input %d)\n",
                    __func__, user_pfc_mode);
                return (EINVAL);
        }

        status = ice_aq_set_pfc_mode(hw, aq_pfc_mode, NULL);
        if (status == ICE_ERR_NOT_SUPPORTED) {
                device_printf(dev,
                    "%s: Failed to set PFC mode; DCB not supported\n",
                    __func__);
                return (ENODEV);
        }
        if (status) {
                device_printf(dev,
                    "%s: Failed to set PFC mode; status %s, aq_err %s\n",
                    __func__, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        /* Reset settings to default when mode is changed */
        ice_set_default_local_mib_settings(sc);
        /* Cache current settings and reconfigure */
        local_dcbx_cfg->pfc_mode = user_pfc_mode;
        ice_do_dcb_reconfig(sc, false);

        return (0);
}

#define ICE_SYSCTL_HELP_SET_LINK_ACTIVE \
"\nKeep link active after setting interface down:" \
"\n\t0 - disable" \
"\n\t1 - enable"

/**
 * ice_sysctl_set_link_active
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Set the link_active_on_if_down sysctl flag.
 */
static int
ice_sysctl_set_link_active(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        bool mode;
        int ret;

        UNREFERENCED_PARAMETER(arg2);

        mode = ice_test_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN);

        ret = sysctl_handle_bool(oidp, &mode, 0, req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        if (mode)
                ice_set_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN);
        else
                ice_clear_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN);

        return (0);
}

/**
 * ice_sysctl_debug_set_link
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Set link up/down in debug session.
 */
static int
ice_sysctl_debug_set_link(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        bool mode;
        int ret;

        UNREFERENCED_PARAMETER(arg2);

        ret = sysctl_handle_bool(oidp, &mode, 0, req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        ice_set_link(sc, mode != 0);

        return (0);
}

/**
 * ice_add_device_sysctls - add device specific dynamic sysctls
 * @sc: device private structure
 *
 * Add per-device dynamic sysctls which show device configuration or enable
 * configuring device functionality. For tunable values which can be set prior
 * to load, see ice_add_device_tunables.
 *
 * This function depends on the sysctl layout setup by ice_add_device_tunables,
 * and likely should be called near the end of the attach process.
 */
void
ice_add_device_sysctls(struct ice_softc *sc)
{
        struct sysctl_oid *hw_node;
        device_t dev = sc->dev;

        struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
        struct sysctl_oid_list *ctx_list =
            SYSCTL_CHILDREN(device_get_sysctl_tree(dev));

        SYSCTL_ADD_PROC(ctx, ctx_list,
            OID_AUTO, "fw_version", CTLTYPE_STRING | CTLFLAG_RD,
            sc, 0, ice_sysctl_show_fw, "A", "Firmware version");

        if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_HAS_PBA)) {
                SYSCTL_ADD_PROC(ctx, ctx_list,
                    OID_AUTO, "pba_number", CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
                    ice_sysctl_pba_number, "A", "Product Board Assembly Number");
        }
        if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_TEMP_SENSOR)) {
                SYSCTL_ADD_PROC(ctx, ctx_list,
                    OID_AUTO, "temp", CTLTYPE_S8 | CTLFLAG_RD,
                    sc, 0, ice_sysctl_temperature, "CU",
                    "Device temperature in degrees Celcius (C)");
        }

        SYSCTL_ADD_PROC(ctx, ctx_list,
            OID_AUTO, "ddp_version", CTLTYPE_STRING | CTLFLAG_RD,
            sc, 0, ice_sysctl_pkg_version, "A", "Active DDP package name and version");

        SYSCTL_ADD_PROC(ctx, ctx_list,
            OID_AUTO, "current_speed", CTLTYPE_STRING | CTLFLAG_RD,
            sc, 0, ice_sysctl_current_speed, "A", "Current Port Link Speed");

        SYSCTL_ADD_PROC(ctx, ctx_list,
            OID_AUTO, "requested_fec", CTLTYPE_STRING | CTLFLAG_RW,
            sc, 0, ice_sysctl_fec_config, "A", ICE_SYSCTL_HELP_FEC_CONFIG);

        SYSCTL_ADD_PROC(ctx, ctx_list,
            OID_AUTO, "negotiated_fec", CTLTYPE_STRING | CTLFLAG_RD,
            sc, 0, ice_sysctl_negotiated_fec, "A", "Current Negotiated FEC mode");

        SYSCTL_ADD_PROC(ctx, ctx_list,
            OID_AUTO, "fc", CTLTYPE_STRING | CTLFLAG_RW,
            sc, 0, ice_sysctl_fc_config, "A", ICE_SYSCTL_HELP_FC_CONFIG);

        SYSCTL_ADD_PROC(ctx, ctx_list,
            OID_AUTO, "advertise_speed", CTLTYPE_U16 | CTLFLAG_RW,
            sc, 0, ice_sysctl_advertise_speed, "SU", ICE_SYSCTL_HELP_ADVERTISE_SPEED);

        SYSCTL_ADD_PROC(ctx, ctx_list,
            OID_AUTO, "fw_lldp_agent", CTLTYPE_U8 | CTLFLAG_RWTUN,
            sc, 0, ice_sysctl_fw_lldp_agent, "CU", ICE_SYSCTL_HELP_FW_LLDP_AGENT);

        SYSCTL_ADD_PROC(ctx, ctx_list,
            OID_AUTO, "ets_min_rate", CTLTYPE_STRING | CTLFLAG_RW,
            sc, 0, ice_sysctl_ets_min_rate, "A", ICE_SYSCTL_HELP_ETS_MIN_RATE);

        SYSCTL_ADD_PROC(ctx, ctx_list,
            OID_AUTO, "up2tc_map", CTLTYPE_STRING | CTLFLAG_RW,
            sc, 0, ice_sysctl_up2tc_map, "A", ICE_SYSCTL_HELP_UP2TC_MAP);

        SYSCTL_ADD_PROC(ctx, ctx_list,
            OID_AUTO, "pfc", CTLTYPE_U8 | CTLFLAG_RW,
            sc, 0, ice_sysctl_pfc_config, "CU", ICE_SYSCTL_HELP_PFC_CONFIG);

        SYSCTL_ADD_PROC(ctx, ctx_list,
            OID_AUTO, "pfc_mode", CTLTYPE_U8 | CTLFLAG_RWTUN,
            sc, 0, ice_sysctl_pfc_mode, "CU", ICE_SYSCTL_HELP_PFC_MODE);

        SYSCTL_ADD_PROC(ctx, ctx_list,
            OID_AUTO, "allow_no_fec_modules_in_auto",
            CTLTYPE_U8 | CTLFLAG_RWTUN | CTLFLAG_MPSAFE,
            sc, 0, ice_sysctl_allow_no_fec_mod_in_auto, "CU",
            "Allow \"No FEC\" mode in FEC auto-negotiation");

        SYSCTL_ADD_PROC(ctx, ctx_list,
            OID_AUTO, "link_active_on_if_down", CTLTYPE_U8 | CTLFLAG_RWTUN,
            sc, 0, ice_sysctl_set_link_active, "CU", ICE_SYSCTL_HELP_SET_LINK_ACTIVE);

        SYSCTL_ADD_PROC(ctx, ctx_list,
            OID_AUTO, "create_mirror_interface", CTLTYPE_STRING | CTLFLAG_RW,
            sc, 0, ice_sysctl_create_mirror_interface, "A", "");

        SYSCTL_ADD_PROC(ctx, ctx_list,
            OID_AUTO, "destroy_mirror_interface", CTLTYPE_STRING | CTLFLAG_RW,
            sc, 0, ice_sysctl_destroy_mirror_interface, "A", "");

        ice_add_dscp2tc_map_sysctls(sc, ctx, ctx_list);

        /* Differentiate software and hardware statistics, by keeping hw stats
         * in their own node. This isn't in ice_add_device_tunables, because
         * we won't have any CTLFLAG_TUN sysctls under this node.
         */
        hw_node = SYSCTL_ADD_NODE(ctx, ctx_list, OID_AUTO, "hw", CTLFLAG_RD,
                                  NULL, "Port Hardware Statistics");

        ice_add_sysctls_mac_stats(ctx, hw_node, sc);

        /* Add the main PF VSI stats now. Other VSIs will add their own stats
         * during creation
         */
        ice_add_vsi_sysctls(&sc->pf_vsi);

        /* Add sysctls related to debugging the device driver. This includes
         * sysctls which display additional internal driver state for use in
         * understanding what is happening within the driver.
         */
        ice_add_debug_sysctls(sc);
}

/**
 * @enum hmc_error_type
 * @brief enumeration of HMC errors
 *
 * Enumeration defining the possible HMC errors that might occur.
 */
enum hmc_error_type {
        HMC_ERR_PMF_INVALID = 0,
        HMC_ERR_VF_IDX_INVALID = 1,
        HMC_ERR_VF_PARENT_PF_INVALID = 2,
        /* 3 is reserved */
        HMC_ERR_INDEX_TOO_BIG = 4,
        HMC_ERR_ADDRESS_TOO_LARGE = 5,
        HMC_ERR_SEGMENT_DESC_INVALID = 6,
        HMC_ERR_SEGMENT_DESC_TOO_SMALL = 7,
        HMC_ERR_PAGE_DESC_INVALID = 8,
        HMC_ERR_UNSUPPORTED_REQUEST_COMPLETION = 9,
        /* 10 is reserved */
        HMC_ERR_INVALID_OBJECT_TYPE = 11,
        /* 12 is reserved */
};

/**
 * ice_log_hmc_error - Log an HMC error message
 * @hw: device hw structure
 * @dev: the device to pass to device_printf()
 *
 * Log a message when an HMC error interrupt is triggered.
 */
void
ice_log_hmc_error(struct ice_hw *hw, device_t dev)
{
        u32 info, data;
        u8 index, errtype, objtype;
        bool isvf;

        info = rd32(hw, PFHMC_ERRORINFO);
        data = rd32(hw, PFHMC_ERRORDATA);

        index = (u8)(info & PFHMC_ERRORINFO_PMF_INDEX_M);
        errtype = (u8)((info & PFHMC_ERRORINFO_HMC_ERROR_TYPE_M) >>
                       PFHMC_ERRORINFO_HMC_ERROR_TYPE_S);
        objtype = (u8)((info & PFHMC_ERRORINFO_HMC_OBJECT_TYPE_M) >>
                       PFHMC_ERRORINFO_HMC_OBJECT_TYPE_S);

        isvf = info & PFHMC_ERRORINFO_PMF_ISVF_M;

        device_printf(dev, "%s HMC Error detected on PMF index %d:\n",
                      isvf ? "VF" : "PF", index);

        device_printf(dev, "error type %d, object type %d, data 0x%08x\n",
                      errtype, objtype, data);

        switch (errtype) {
        case HMC_ERR_PMF_INVALID:
                device_printf(dev, "Private Memory Function is not valid\n");
                break;
        case HMC_ERR_VF_IDX_INVALID:
                device_printf(dev, "Invalid Private Memory Function index for PE enabled VF\n");
                break;
        case HMC_ERR_VF_PARENT_PF_INVALID:
                device_printf(dev, "Invalid parent PF for PE enabled VF\n");
                break;
        case HMC_ERR_INDEX_TOO_BIG:
                device_printf(dev, "Object index too big\n");
                break;
        case HMC_ERR_ADDRESS_TOO_LARGE:
                device_printf(dev, "Address extends beyond segment descriptor limit\n");
                break;
        case HMC_ERR_SEGMENT_DESC_INVALID:
                device_printf(dev, "Segment descriptor is invalid\n");
                break;
        case HMC_ERR_SEGMENT_DESC_TOO_SMALL:
                device_printf(dev, "Segment descriptor is too small\n");
                break;
        case HMC_ERR_PAGE_DESC_INVALID:
                device_printf(dev, "Page descriptor is invalid\n");
                break;
        case HMC_ERR_UNSUPPORTED_REQUEST_COMPLETION:
                device_printf(dev, "Unsupported Request completion received from PCIe\n");
                break;
        case HMC_ERR_INVALID_OBJECT_TYPE:
                device_printf(dev, "Invalid object type\n");
                break;
        default:
                device_printf(dev, "Unknown HMC error\n");
        }

        /* Clear the error indication */
        wr32(hw, PFHMC_ERRORINFO, 0);
}

/**
 * @struct ice_sysctl_info
 * @brief sysctl information
 *
 * Structure used to simplify the process of defining the many similar
 * statistics sysctls.
 */
struct ice_sysctl_info {
        u64             *stat;
        const char      *name;
        const char      *description;
};

/**
 * ice_add_sysctls_eth_stats - Add sysctls for ethernet statistics
 * @ctx: sysctl ctx to use
 * @parent: the parent node to add sysctls under
 * @stats: the ethernet stats structure to source values from
 *
 * Adds statistics sysctls for the ethernet statistics of the MAC or a VSI.
 * Will add them under the parent node specified.
 *
 * Note that tx_errors is only meaningful for VSIs and not the global MAC/PF
 * statistics, so it is not included here. Similarly, rx_discards has different
 * descriptions for VSIs and MAC/PF stats, so it is also not included here.
 */
void
ice_add_sysctls_eth_stats(struct sysctl_ctx_list *ctx,
                          struct sysctl_oid *parent,
                          struct ice_eth_stats *stats)
{
        const struct ice_sysctl_info ctls[] = {
                /* Rx Stats */
                { &stats->rx_bytes, "good_octets_rcvd", "Good Octets Received" },
                { &stats->rx_unicast, "ucast_pkts_rcvd", "Unicast Packets Received" },
                { &stats->rx_multicast, "mcast_pkts_rcvd", "Multicast Packets Received" },
                { &stats->rx_broadcast, "bcast_pkts_rcvd", "Broadcast Packets Received" },
                /* Tx Stats */
                { &stats->tx_bytes, "good_octets_txd", "Good Octets Transmitted" },
                { &stats->tx_unicast, "ucast_pkts_txd", "Unicast Packets Transmitted" },
                { &stats->tx_multicast, "mcast_pkts_txd", "Multicast Packets Transmitted" },
                { &stats->tx_broadcast, "bcast_pkts_txd", "Broadcast Packets Transmitted" },
                /* End */
                { 0, 0, 0 }
        };

        struct sysctl_oid_list *parent_list = SYSCTL_CHILDREN(parent);

        const struct ice_sysctl_info *entry = ctls;
        while (entry->stat != 0) {
                SYSCTL_ADD_U64(ctx, parent_list, OID_AUTO, entry->name,
                               CTLFLAG_RD | CTLFLAG_STATS, entry->stat, 0,
                               entry->description);
                entry++;
        }
}

/**
 * ice_sysctl_tx_cso_stat - Display Tx checksum offload statistic
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: Tx CSO stat to read
 * @req: sysctl request pointer
 *
 * On read: Sums the per-queue Tx CSO stat and displays it.
 */
static int
ice_sysctl_tx_cso_stat(SYSCTL_HANDLER_ARGS)
{
        struct ice_vsi *vsi = (struct ice_vsi *)arg1;
        enum ice_tx_cso_stat type = (enum ice_tx_cso_stat)arg2;
        u64 stat = 0;
        int i;

        if (ice_driver_is_detaching(vsi->sc))
                return (ESHUTDOWN);

        /* Check that the type is valid */
        if (type >= ICE_CSO_STAT_TX_COUNT)
                return (EDOOFUS);

        /* Sum the stat for each of the Tx queues */
        for (i = 0; i < vsi->num_tx_queues; i++)
                stat += vsi->tx_queues[i].stats.cso[type];

        return sysctl_handle_64(oidp, NULL, stat, req);
}

/**
 * ice_sysctl_rx_cso_stat - Display Rx checksum offload statistic
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: Rx CSO stat to read
 * @req: sysctl request pointer
 *
 * On read: Sums the per-queue Rx CSO stat and displays it.
 */
static int
ice_sysctl_rx_cso_stat(SYSCTL_HANDLER_ARGS)
{
        struct ice_vsi *vsi = (struct ice_vsi *)arg1;
        enum ice_rx_cso_stat type = (enum ice_rx_cso_stat)arg2;
        u64 stat = 0;
        int i;

        if (ice_driver_is_detaching(vsi->sc))
                return (ESHUTDOWN);

        /* Check that the type is valid */
        if (type >= ICE_CSO_STAT_RX_COUNT)
                return (EDOOFUS);

        /* Sum the stat for each of the Rx queues */
        for (i = 0; i < vsi->num_rx_queues; i++)
                stat += vsi->rx_queues[i].stats.cso[type];

        return sysctl_handle_64(oidp, NULL, stat, req);
}

/**
 * ice_sysctl_rx_errors_stat - Display aggregate of Rx errors
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Sums current values of Rx error statistics and
 * displays it.
 */
static int
ice_sysctl_rx_errors_stat(SYSCTL_HANDLER_ARGS)
{
        struct ice_vsi *vsi = (struct ice_vsi *)arg1;
        struct ice_hw_port_stats *hs = &vsi->sc->stats.cur;
        u64 stat = 0;
        int i, type;

        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(vsi->sc))
                return (ESHUTDOWN);

        stat += hs->rx_undersize;
        stat += hs->rx_fragments;
        stat += hs->rx_oversize;
        stat += hs->rx_jabber;
        stat += hs->crc_errors;
        stat += hs->illegal_bytes;

        /* Checksum error stats */
        for (i = 0; i < vsi->num_rx_queues; i++)
                for (type = ICE_CSO_STAT_RX_IP4_ERR;
                     type < ICE_CSO_STAT_RX_COUNT;
                     type++)
                        stat += vsi->rx_queues[i].stats.cso[type];

        return sysctl_handle_64(oidp, NULL, stat, req);
}

/**
 * @struct ice_rx_cso_stat_info
 * @brief sysctl information for an Rx checksum offload statistic
 *
 * Structure used to simplify the process of defining the checksum offload
 * statistics.
 */
struct ice_rx_cso_stat_info {
        enum ice_rx_cso_stat    type;
        const char              *name;
        const char              *description;
};

/**
 * @struct ice_tx_cso_stat_info
 * @brief sysctl information for a Tx checksum offload statistic
 *
 * Structure used to simplify the process of defining the checksum offload
 * statistics.
 */
struct ice_tx_cso_stat_info {
        enum ice_tx_cso_stat    type;
        const char              *name;
        const char              *description;
};

/**
 * ice_add_sysctls_sw_stats - Add sysctls for software statistics
 * @vsi: pointer to the VSI to add sysctls for
 * @ctx: sysctl ctx to use
 * @parent: the parent node to add sysctls under
 *
 * Add statistics sysctls for software tracked statistics of a VSI.
 *
 * Currently this only adds checksum offload statistics, but more counters may
 * be added in the future.
 */
static void
ice_add_sysctls_sw_stats(struct ice_vsi *vsi,
                         struct sysctl_ctx_list *ctx,
                         struct sysctl_oid *parent)
{
        struct sysctl_oid *cso_node;
        struct sysctl_oid_list *cso_list;

        /* Tx CSO Stats */
        const struct ice_tx_cso_stat_info tx_ctls[] = {
                { ICE_CSO_STAT_TX_TCP, "tx_tcp", "Transmit TCP Packets marked for HW checksum" },
                { ICE_CSO_STAT_TX_UDP, "tx_udp", "Transmit UDP Packets marked for HW checksum" },
                { ICE_CSO_STAT_TX_SCTP, "tx_sctp", "Transmit SCTP Packets marked for HW checksum" },
                { ICE_CSO_STAT_TX_IP4, "tx_ip4", "Transmit IPv4 Packets marked for HW checksum" },
                { ICE_CSO_STAT_TX_IP6, "tx_ip6", "Transmit IPv6 Packets marked for HW checksum" },
                { ICE_CSO_STAT_TX_L3_ERR, "tx_l3_err", "Transmit packets that driver failed to set L3 HW CSO bits for" },
                { ICE_CSO_STAT_TX_L4_ERR, "tx_l4_err", "Transmit packets that driver failed to set L4 HW CSO bits for" },
                /* End */
                { ICE_CSO_STAT_TX_COUNT, 0, 0 }
        };

        /* Rx CSO Stats */
        const struct ice_rx_cso_stat_info rx_ctls[] = {
                { ICE_CSO_STAT_RX_IP4_ERR, "rx_ip4_err", "Received packets with invalid IPv4 checksum indicated by HW" },
                { ICE_CSO_STAT_RX_IP6_ERR, "rx_ip6_err", "Received IPv6 packets with extension headers" },
                { ICE_CSO_STAT_RX_L3_ERR, "rx_l3_err", "Received packets with an unexpected invalid L3 checksum indicated by HW" },
                { ICE_CSO_STAT_RX_TCP_ERR, "rx_tcp_err", "Received packets with invalid TCP checksum indicated by HW" },
                { ICE_CSO_STAT_RX_UDP_ERR, "rx_udp_err", "Received packets with invalid UDP checksum indicated by HW" },
                { ICE_CSO_STAT_RX_SCTP_ERR, "rx_sctp_err", "Received packets with invalid SCTP checksum indicated by HW" },
                { ICE_CSO_STAT_RX_L4_ERR, "rx_l4_err", "Received packets with an unexpected invalid L4 checksum indicated by HW" },
                /* End */
                { ICE_CSO_STAT_RX_COUNT, 0, 0 }
        };

        struct sysctl_oid_list *parent_list = SYSCTL_CHILDREN(parent);

        /* Add a node for statistics tracked by software. */
        cso_node = SYSCTL_ADD_NODE(ctx, parent_list, OID_AUTO, "cso", CTLFLAG_RD,
                                  NULL, "Checksum offload Statistics");
        cso_list = SYSCTL_CHILDREN(cso_node);

        const struct ice_tx_cso_stat_info *tx_entry = tx_ctls;
        while (tx_entry->name && tx_entry->description) {
                SYSCTL_ADD_PROC(ctx, cso_list, OID_AUTO, tx_entry->name,
                                CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_STATS,
                                vsi, tx_entry->type, ice_sysctl_tx_cso_stat, "QU",
                                tx_entry->description);
                tx_entry++;
        }

        const struct ice_rx_cso_stat_info *rx_entry = rx_ctls;
        while (rx_entry->name && rx_entry->description) {
                SYSCTL_ADD_PROC(ctx, cso_list, OID_AUTO, rx_entry->name,
                                CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_STATS,
                                vsi, rx_entry->type, ice_sysctl_rx_cso_stat, "QU",
                                rx_entry->description);
                rx_entry++;
        }
}

/**
 * ice_add_vsi_sysctls - Add sysctls for a VSI
 * @vsi: pointer to VSI structure
 *
 * Add various sysctls for a given VSI.
 */
void
ice_add_vsi_sysctls(struct ice_vsi *vsi)
{
        struct sysctl_ctx_list *ctx = &vsi->ctx;
        struct sysctl_oid *hw_node, *sw_node;
        struct sysctl_oid_list *vsi_list, *hw_list;

        vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);

        /* Keep hw stats in their own node. */
        hw_node = SYSCTL_ADD_NODE(ctx, vsi_list, OID_AUTO, "hw", CTLFLAG_RD,
                                  NULL, "VSI Hardware Statistics");
        hw_list = SYSCTL_CHILDREN(hw_node);

        /* Add the ethernet statistics for this VSI */
        ice_add_sysctls_eth_stats(ctx, hw_node, &vsi->hw_stats.cur);

        SYSCTL_ADD_U64(ctx, hw_list, OID_AUTO, "rx_discards",
                        CTLFLAG_RD | CTLFLAG_STATS, &vsi->hw_stats.cur.rx_discards,
                        0, "Discarded Rx Packets (see rx_errors or rx_no_desc)");

        SYSCTL_ADD_PROC(ctx, hw_list, OID_AUTO, "rx_errors",
                        CTLTYPE_U64 | CTLFLAG_RD | CTLFLAG_STATS,
                        vsi, 0, ice_sysctl_rx_errors_stat, "QU",
                        "Aggregate of all Rx errors");

        SYSCTL_ADD_U64(ctx, hw_list, OID_AUTO, "rx_no_desc",
                       CTLFLAG_RD | CTLFLAG_STATS, &vsi->hw_stats.cur.rx_no_desc,
                       0, "Rx Packets Discarded Due To Lack Of Descriptors");

        SYSCTL_ADD_U64(ctx, hw_list, OID_AUTO, "tx_errors",
                        CTLFLAG_RD | CTLFLAG_STATS, &vsi->hw_stats.cur.tx_errors,
                        0, "Tx Packets Discarded Due To Error");

        /* Add a node for statistics tracked by software. */
        sw_node = SYSCTL_ADD_NODE(ctx, vsi_list, OID_AUTO, "sw", CTLFLAG_RD,
                                  NULL, "VSI Software Statistics");

        ice_add_sysctls_sw_stats(vsi, ctx, sw_node);
}

/**
 * ice_add_sysctls_mac_pfc_one_stat - Add sysctl node for a PFC statistic
 * @ctx: sysctl ctx to use
 * @parent_list: parent sysctl list to add sysctls under
 * @pfc_stat_location: address of statistic for sysctl to display
 * @node_name: Name for statistic node
 * @descr: Description used for nodes added in this function
 *
 * A helper function for ice_add_sysctls_mac_pfc_stats that adds a node
 * for a stat and leaves for each traffic class for that stat.
 */
static void
ice_add_sysctls_mac_pfc_one_stat(struct sysctl_ctx_list *ctx,
                                 struct sysctl_oid_list *parent_list,
                                 u64* pfc_stat_location,
                                 const char *node_name,
                                 const char *descr)
{
        struct sysctl_oid_list *node_list;
        struct sysctl_oid *node;
        struct sbuf *namebuf, *descbuf;

        node = SYSCTL_ADD_NODE(ctx, parent_list, OID_AUTO, node_name, CTLFLAG_RD,
                                   NULL, descr);
        node_list = SYSCTL_CHILDREN(node);

        namebuf = sbuf_new_auto();
        descbuf = sbuf_new_auto();
        for (int i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++) {
                sbuf_clear(namebuf);
                sbuf_clear(descbuf);

                sbuf_printf(namebuf, "%d", i);
                sbuf_printf(descbuf, "%s for TC %d", descr, i);

                sbuf_finish(namebuf);
                sbuf_finish(descbuf);

                SYSCTL_ADD_U64(ctx, node_list, OID_AUTO, sbuf_data(namebuf),
                        CTLFLAG_RD | CTLFLAG_STATS, &pfc_stat_location[i], 0,
                        sbuf_data(descbuf));
        }

        sbuf_delete(namebuf);
        sbuf_delete(descbuf);
}

/**
 * ice_add_sysctls_mac_pfc_stats - Add sysctls for MAC PFC statistics
 * @ctx: the sysctl ctx to use
 * @parent: parent node to add the sysctls under
 * @stats: the hw ports stat structure to pull values from
 *
 * Add global Priority Flow Control MAC statistics sysctls. These are
 * structured as a node with the PFC statistic, where there are eight
 * nodes for each traffic class.
 */
static void
ice_add_sysctls_mac_pfc_stats(struct sysctl_ctx_list *ctx,
                              struct sysctl_oid *parent,
                              struct ice_hw_port_stats *stats)
{
        struct sysctl_oid_list *parent_list;

        parent_list = SYSCTL_CHILDREN(parent);

        ice_add_sysctls_mac_pfc_one_stat(ctx, parent_list, stats->priority_xon_rx,
            "p_xon_recvd", "PFC XON received");
        ice_add_sysctls_mac_pfc_one_stat(ctx, parent_list, stats->priority_xoff_rx,
            "p_xoff_recvd", "PFC XOFF received");
        ice_add_sysctls_mac_pfc_one_stat(ctx, parent_list, stats->priority_xon_tx,
            "p_xon_txd", "PFC XON transmitted");
        ice_add_sysctls_mac_pfc_one_stat(ctx, parent_list, stats->priority_xoff_tx,
            "p_xoff_txd", "PFC XOFF transmitted");
        ice_add_sysctls_mac_pfc_one_stat(ctx, parent_list, stats->priority_xon_2_xoff,
            "p_xon2xoff", "PFC XON to XOFF transitions");
}

/**
 * ice_add_sysctls_mac_stats - Add sysctls for global MAC statistics
 * @ctx: the sysctl ctx to use
 * @parent: parent node to add the sysctls under
 * @sc: device private structure
 *
 * Add global MAC statistics sysctls.
 */
void
ice_add_sysctls_mac_stats(struct sysctl_ctx_list *ctx,
                          struct sysctl_oid *parent,
                          struct ice_softc *sc)
{
        struct sysctl_oid *mac_node;
        struct sysctl_oid_list *parent_list, *mac_list;
        struct ice_hw_port_stats *stats = &sc->stats.cur;

        parent_list = SYSCTL_CHILDREN(parent);

        mac_node = SYSCTL_ADD_NODE(ctx, parent_list, OID_AUTO, "mac", CTLFLAG_RD,
                                   NULL, "Mac Hardware Statistics");
        mac_list = SYSCTL_CHILDREN(mac_node);

        /* Add the ethernet statistics common to VSI and MAC */
        ice_add_sysctls_eth_stats(ctx, mac_node, &stats->eth);

        /* Add PFC stats that add per-TC counters */
        ice_add_sysctls_mac_pfc_stats(ctx, mac_node, stats);

        const struct ice_sysctl_info ctls[] = {
                /* Packet Reception Stats */
                {&stats->rx_size_64, "rx_frames_64", "64 byte frames received"},
                {&stats->rx_size_127, "rx_frames_65_127", "65-127 byte frames received"},
                {&stats->rx_size_255, "rx_frames_128_255", "128-255 byte frames received"},
                {&stats->rx_size_511, "rx_frames_256_511", "256-511 byte frames received"},
                {&stats->rx_size_1023, "rx_frames_512_1023", "512-1023 byte frames received"},
                {&stats->rx_size_1522, "rx_frames_1024_1522", "1024-1522 byte frames received"},
                {&stats->rx_size_big, "rx_frames_big", "1523-9522 byte frames received"},
                {&stats->rx_undersize, "rx_undersize", "Undersized packets received"},
                {&stats->rx_fragments, "rx_fragmented", "Fragmented packets received"},
                {&stats->rx_jabber, "rx_jabber", "Received Jabber"},
                {&stats->eth.rx_discards, "rx_discards",
                    "Discarded Rx Packets by Port (shortage of storage space)"},
                /* Packet Transmission Stats */
                {&stats->tx_size_64, "tx_frames_64", "64 byte frames transmitted"},
                {&stats->tx_size_127, "tx_frames_65_127", "65-127 byte frames transmitted"},
                {&stats->tx_size_255, "tx_frames_128_255", "128-255 byte frames transmitted"},
                {&stats->tx_size_511, "tx_frames_256_511", "256-511 byte frames transmitted"},
                {&stats->tx_size_1023, "tx_frames_512_1023", "512-1023 byte frames transmitted"},
                {&stats->tx_size_1522, "tx_frames_1024_1522", "1024-1522 byte frames transmitted"},
                {&stats->tx_size_big, "tx_frames_big", "1523-9522 byte frames transmitted"},
                {&stats->tx_dropped_link_down, "tx_dropped", "Tx Dropped Due To Link Down"},
                /* Flow control */
                {&stats->link_xon_tx, "xon_txd", "Link XON transmitted"},
                {&stats->link_xon_rx, "xon_recvd", "Link XON received"},
                {&stats->link_xoff_tx, "xoff_txd", "Link XOFF transmitted"},
                {&stats->link_xoff_rx, "xoff_recvd", "Link XOFF received"},
                /* Other */
                {&stats->crc_errors, "crc_errors", "CRC Errors"},
                {&stats->illegal_bytes, "illegal_bytes", "Illegal Byte Errors"},
                {&stats->mac_local_faults, "local_faults", "MAC Local Faults"},
                {&stats->mac_remote_faults, "remote_faults", "MAC Remote Faults"},
                /* End */
                { 0, 0, 0 }
        };

        const struct ice_sysctl_info *entry = ctls;
        while (entry->stat != 0) {
                SYSCTL_ADD_U64(ctx, mac_list, OID_AUTO, entry->name,
                        CTLFLAG_RD | CTLFLAG_STATS, entry->stat, 0,
                        entry->description);
                entry++;
        }
        /* Port oversize packet stats */
        SYSCTL_ADD_U64(ctx, mac_list, OID_AUTO, "rx_oversized",
                       CTLFLAG_RD | CTLFLAG_STATS, &sc->soft_stats.rx_roc_error,
                       0, "Oversized packets received");

}

/**
 * ice_configure_misc_interrupts - enable 'other' interrupt causes
 * @sc: pointer to device private softc
 *
 * Enable various "other" interrupt causes, and associate them to interrupt 0,
 * which is our administrative interrupt.
 */
void
ice_configure_misc_interrupts(struct ice_softc *sc)
{
        struct ice_hw *hw = &sc->hw;
        u32 val;

        /* Read the OICR register to clear it */
        rd32(hw, PFINT_OICR);

        /* Enable useful "other" interrupt causes */
        val = (PFINT_OICR_ECC_ERR_M |
               PFINT_OICR_MAL_DETECT_M |
               PFINT_OICR_GRST_M |
               PFINT_OICR_PCI_EXCEPTION_M |
               PFINT_OICR_VFLR_M |
               PFINT_OICR_HMC_ERR_M |
               PFINT_OICR_PE_CRITERR_M);

        wr32(hw, PFINT_OICR_ENA, val);

        /* Note that since we're using MSI-X index 0, and ITR index 0, we do
         * not explicitly program them when writing to the PFINT_*_CTL
         * registers. Nevertheless, these writes are associating the
         * interrupts with the ITR 0 vector
         */

        /* Associate the OICR interrupt with ITR 0, and enable it */
        wr32(hw, PFINT_OICR_CTL, PFINT_OICR_CTL_CAUSE_ENA_M);

        /* Associate the Mailbox interrupt with ITR 0, and enable it */
        wr32(hw, PFINT_MBX_CTL, PFINT_MBX_CTL_CAUSE_ENA_M);

        /* Associate the SB Queue interrupt with ITR 0, and enable it */
        wr32(hw, PFINT_SB_CTL, PFINT_SB_CTL_CAUSE_ENA_M);

        /* Associate the AdminQ interrupt with ITR 0, and enable it */
        wr32(hw, PFINT_FW_CTL, PFINT_FW_CTL_CAUSE_ENA_M);
}

/**
 * ice_filter_is_mcast - Check if info is a multicast filter
 * @vsi: vsi structure addresses are targeted towards
 * @info: filter info
 *
 * @returns true if the provided info is a multicast filter, and false
 * otherwise.
 */
static bool
ice_filter_is_mcast(struct ice_vsi *vsi, struct ice_fltr_info *info)
{
        const u8 *addr = info->l_data.mac.mac_addr;

        /*
         * Check if this info matches a multicast filter added by
         * ice_add_mac_to_list
         */
        if ((info->flag == ICE_FLTR_TX) &&
            (info->src_id == ICE_SRC_ID_VSI) &&
            (info->lkup_type == ICE_SW_LKUP_MAC) &&
            (info->vsi_handle == vsi->idx) &&
            ETHER_IS_MULTICAST(addr) && !ETHER_IS_BROADCAST(addr))
                return true;

        return false;
}

/**
 * @struct ice_mcast_sync_data
 * @brief data used by ice_sync_one_mcast_filter function
 *
 * Structure used to store data needed for processing by the
 * ice_sync_one_mcast_filter. This structure contains a linked list of filters
 * to be added, an error indication, and a pointer to the device softc.
 */
struct ice_mcast_sync_data {
        struct ice_list_head add_list;
        struct ice_softc *sc;
        int err;
};

/**
 * ice_sync_one_mcast_filter - Check if we need to program the filter
 * @p: void pointer to algorithm data
 * @sdl: link level socket address
 * @count: unused count value
 *
 * Called by if_foreach_llmaddr to operate on each filter in the ifp filter
 * list. For the given address, search our internal list to see if we have
 * found the filter. If not, add it to our list of filters that need to be
 * programmed.
 *
 * @returns (1) if we've actually setup the filter to be added
 */
static u_int
ice_sync_one_mcast_filter(void *p, struct sockaddr_dl *sdl,
                          u_int __unused count)
{
        struct ice_mcast_sync_data *data = (struct ice_mcast_sync_data *)p;
        struct ice_softc *sc = data->sc;
        struct ice_hw *hw = &sc->hw;
        struct ice_switch_info *sw = hw->switch_info;
        const u8 *sdl_addr = (const u8 *)LLADDR(sdl);
        struct ice_fltr_mgmt_list_entry *itr;
        struct ice_list_head *rules;
        int err;

        rules = &sw->recp_list[ICE_SW_LKUP_MAC].filt_rules;

        /*
         * If a previous filter already indicated an error, there is no need
         * for us to finish processing the rest of the filters.
         */
        if (data->err)
                return (0);

        /* See if this filter has already been programmed */
        LIST_FOR_EACH_ENTRY(itr, rules, ice_fltr_mgmt_list_entry, list_entry) {
                struct ice_fltr_info *info = &itr->fltr_info;
                const u8 *addr = info->l_data.mac.mac_addr;

                /* Only check multicast filters */
                if (!ice_filter_is_mcast(&sc->pf_vsi, info))
                        continue;

                /*
                 * If this filter matches, mark the internal filter as
                 * "found", and exit.
                 */
                if (bcmp(addr, sdl_addr, ETHER_ADDR_LEN) == 0) {
                        itr->marker = ICE_FLTR_FOUND;
                        return (1);
                }
        }

        /*
         * If we failed to locate the filter in our internal list, we need to
         * place it into our add list.
         */
        err = ice_add_mac_to_list(&sc->pf_vsi, &data->add_list, sdl_addr,
                                  ICE_FWD_TO_VSI);
        if (err) {
                device_printf(sc->dev,
                              "Failed to place MAC %6D onto add list, err %s\n",
                              sdl_addr, ":", ice_err_str(err));
                data->err = err;

                return (0);
        }

        return (1);
}

/**
 * ice_sync_multicast_filters - Synchronize OS and internal filter list
 * @sc: device private structure
 *
 * Called in response to SIOCDELMULTI to synchronize the operating system
 * multicast address list with the internal list of filters programmed to
 * firmware.
 *
 * Works in one phase to find added and deleted filters using a marker bit on
 * the internal list.
 *
 * First, a loop over the internal list clears the marker bit. Second, for
 * each filter in the ifp list is checked. If we find it in the internal list,
 * the marker bit is set. Otherwise, the filter is added to the add list.
 * Third, a loop over the internal list determines if any filters have not
 * been found. Each of these is added to the delete list. Finally, the add and
 * delete lists are programmed to firmware to update the filters.
 *
 * @returns zero on success or an integer error code on failure.
 */
int
ice_sync_multicast_filters(struct ice_softc *sc)
{
        struct ice_hw *hw = &sc->hw;
        struct ice_switch_info *sw = hw->switch_info;
        struct ice_fltr_mgmt_list_entry *itr;
        struct ice_mcast_sync_data data = {};
        struct ice_list_head *rules, remove_list;
        int status;
        int err = 0;

        INIT_LIST_HEAD(&data.add_list);
        INIT_LIST_HEAD(&remove_list);
        data.sc = sc;
        data.err = 0;

        rules = &sw->recp_list[ICE_SW_LKUP_MAC].filt_rules;

        /* Acquire the lock for the entire duration */
        ice_acquire_lock(&sw->recp_list[ICE_SW_LKUP_MAC].filt_rule_lock);

        /* (1) Reset the marker state for all filters */
        LIST_FOR_EACH_ENTRY(itr, rules, ice_fltr_mgmt_list_entry, list_entry)
                itr->marker = ICE_FLTR_NOT_FOUND;

        /* (2) determine which filters need to be added and removed */
        if_foreach_llmaddr(sc->ifp, ice_sync_one_mcast_filter, (void *)&data);
        if (data.err) {
                /* ice_sync_one_mcast_filter already prints an error */
                err = data.err;
                ice_release_lock(&sw->recp_list[ICE_SW_LKUP_MAC].filt_rule_lock);
                goto free_filter_lists;
        }

        LIST_FOR_EACH_ENTRY(itr, rules, ice_fltr_mgmt_list_entry, list_entry) {
                struct ice_fltr_info *info = &itr->fltr_info;
                const u8 *addr = info->l_data.mac.mac_addr;

                /* Only check multicast filters */
                if (!ice_filter_is_mcast(&sc->pf_vsi, info))
                        continue;

                /*
                 * If the filter is not marked as found, then it must no
                 * longer be in the ifp address list, so we need to remove it.
                 */
                if (itr->marker == ICE_FLTR_NOT_FOUND) {
                        err = ice_add_mac_to_list(&sc->pf_vsi, &remove_list,
                                                  addr, ICE_FWD_TO_VSI);
                        if (err) {
                                device_printf(sc->dev,
                                              "Failed to place MAC %6D onto remove list, err %s\n",
                                              addr, ":", ice_err_str(err));
                                ice_release_lock(&sw->recp_list[ICE_SW_LKUP_MAC].filt_rule_lock);
                                goto free_filter_lists;
                        }
                }
        }

        ice_release_lock(&sw->recp_list[ICE_SW_LKUP_MAC].filt_rule_lock);

        status = ice_add_mac(hw, &data.add_list);
        if (status) {
                device_printf(sc->dev,
                              "Could not add new MAC filters, err %s aq_err %s\n",
                              ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
                err = (EIO);
                goto free_filter_lists;
        }

        status = ice_remove_mac(hw, &remove_list);
        if (status) {
                device_printf(sc->dev,
                              "Could not remove old MAC filters, err %s aq_err %s\n",
                              ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
                err = (EIO);
                goto free_filter_lists;
        }

free_filter_lists:
        ice_free_fltr_list(&data.add_list);
        ice_free_fltr_list(&remove_list);

        return (err);
}

/**
 * ice_add_vlan_hw_filters - Add multiple VLAN filters for a given VSI
 * @vsi: The VSI to add the filter for
 * @vid: array of VLAN ids to add
 * @length: length of vid array
 *
 * Programs HW filters so that the given VSI will receive the specified VLANs.
 */
int
ice_add_vlan_hw_filters(struct ice_vsi *vsi, u16 *vid, u16 length)
{
        struct ice_hw *hw = &vsi->sc->hw;
        struct ice_list_head vlan_list;
        struct ice_fltr_list_entry *vlan_entries;
        int status;

        MPASS(length > 0);

        INIT_LIST_HEAD(&vlan_list);

        vlan_entries = (struct ice_fltr_list_entry *)
            malloc(sizeof(*vlan_entries) * length, M_ICE, M_NOWAIT | M_ZERO);
        if (!vlan_entries)
                return (ICE_ERR_NO_MEMORY);

        for (u16 i = 0; i < length; i++) {
                vlan_entries[i].fltr_info.lkup_type = ICE_SW_LKUP_VLAN;
                vlan_entries[i].fltr_info.fltr_act = ICE_FWD_TO_VSI;
                vlan_entries[i].fltr_info.flag = ICE_FLTR_TX;
                vlan_entries[i].fltr_info.src_id = ICE_SRC_ID_VSI;
                vlan_entries[i].fltr_info.vsi_handle = vsi->idx;
                vlan_entries[i].fltr_info.l_data.vlan.vlan_id = vid[i];

                LIST_ADD(&vlan_entries[i].list_entry, &vlan_list);
        }

        status = ice_add_vlan(hw, &vlan_list);
        if (!status)
                goto done;

        device_printf(vsi->sc->dev, "Failed to add VLAN filters:\n");
        for (u16 i = 0; i < length; i++) {
                device_printf(vsi->sc->dev,
                    "- vlan %d, status %d\n",
                    vlan_entries[i].fltr_info.l_data.vlan.vlan_id,
                    vlan_entries[i].status);
        }
done:
        free(vlan_entries, M_ICE);
        return (status);
}

/**
 * ice_add_vlan_hw_filter - Add a VLAN filter for a given VSI
 * @vsi: The VSI to add the filter for
 * @vid: VLAN to add
 *
 * Programs a HW filter so that the given VSI will receive the specified VLAN.
 */
int
ice_add_vlan_hw_filter(struct ice_vsi *vsi, u16 vid)
{
        return ice_add_vlan_hw_filters(vsi, &vid, 1);
}

/**
 * ice_remove_vlan_hw_filters - Remove multiple VLAN filters for a given VSI
 * @vsi: The VSI to remove the filters from
 * @vid: array of VLAN ids to remove
 * @length: length of vid array
 *
 * Removes previously programmed HW filters for the specified VSI.
 */
int
ice_remove_vlan_hw_filters(struct ice_vsi *vsi, u16 *vid, u16 length)
{
        struct ice_hw *hw = &vsi->sc->hw;
        struct ice_list_head vlan_list;
        struct ice_fltr_list_entry *vlan_entries;
        int status;

        MPASS(length > 0);

        INIT_LIST_HEAD(&vlan_list);

        vlan_entries = (struct ice_fltr_list_entry *)
            malloc(sizeof(*vlan_entries) * length, M_ICE, M_NOWAIT | M_ZERO);
        if (!vlan_entries)
                return (ICE_ERR_NO_MEMORY);

        for (u16 i = 0; i < length; i++) {
                vlan_entries[i].fltr_info.lkup_type = ICE_SW_LKUP_VLAN;
                vlan_entries[i].fltr_info.fltr_act = ICE_FWD_TO_VSI;
                vlan_entries[i].fltr_info.flag = ICE_FLTR_TX;
                vlan_entries[i].fltr_info.src_id = ICE_SRC_ID_VSI;
                vlan_entries[i].fltr_info.vsi_handle = vsi->idx;
                vlan_entries[i].fltr_info.l_data.vlan.vlan_id = vid[i];

                LIST_ADD(&vlan_entries[i].list_entry, &vlan_list);
        }

        status = ice_remove_vlan(hw, &vlan_list);
        if (!status)
                goto done;

        device_printf(vsi->sc->dev, "Failed to remove VLAN filters:\n");
        for (u16 i = 0; i < length; i++) {
                device_printf(vsi->sc->dev,
                    "- vlan %d, status %d\n",
                    vlan_entries[i].fltr_info.l_data.vlan.vlan_id,
                    vlan_entries[i].status);
        }
done:
        free(vlan_entries, M_ICE);
        return (status);
}

/**
 * ice_remove_vlan_hw_filter - Remove a VLAN filter for a given VSI
 * @vsi: The VSI to remove the filter from
 * @vid: VLAN to remove
 *
 * Removes a previously programmed HW filter for the specified VSI.
 */
int
ice_remove_vlan_hw_filter(struct ice_vsi *vsi, u16 vid)
{
        return ice_remove_vlan_hw_filters(vsi, &vid, 1);
}

#define ICE_SYSCTL_HELP_RX_ITR                  \
"\nControl Rx interrupt throttle rate."         \
"\n\t0-8160 - sets interrupt rate in usecs"     \
"\n\t    -1 - reset the Rx itr to default"

/**
 * ice_sysctl_rx_itr - Display or change the Rx ITR for a VSI
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Displays the current Rx ITR value
 * on write: Sets the Rx ITR value, reconfiguring device if it is up
 */
static int
ice_sysctl_rx_itr(SYSCTL_HANDLER_ARGS)
{
        struct ice_vsi *vsi = (struct ice_vsi *)arg1;
        struct ice_softc *sc = vsi->sc;
        int increment, ret;

        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        ret = sysctl_handle_16(oidp, &vsi->rx_itr, 0, req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        if (vsi->rx_itr < 0)
                vsi->rx_itr = ICE_DFLT_RX_ITR;
        if (vsi->rx_itr > ICE_ITR_MAX)
                vsi->rx_itr = ICE_ITR_MAX;

        /* Assume 2usec increment if it hasn't been loaded yet */
        increment = sc->hw.itr_gran ? : 2;

        /* We need to round the value to the hardware's ITR granularity */
        vsi->rx_itr = (vsi->rx_itr / increment ) * increment;

        /* If the driver has finished initializing, then we need to reprogram
         * the ITR registers now. Otherwise, they will be programmed during
         * driver initialization.
         */
        if (ice_test_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED))
                ice_configure_rx_itr(vsi);

        return (0);
}

#define ICE_SYSCTL_HELP_TX_ITR                  \
"\nControl Tx interrupt throttle rate."         \
"\n\t0-8160 - sets interrupt rate in usecs"     \
"\n\t    -1 - reset the Tx itr to default"

/**
 * ice_sysctl_tx_itr - Display or change the Tx ITR for a VSI
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * On read: Displays the current Tx ITR value
 * on write: Sets the Tx ITR value, reconfiguring device if it is up
 */
static int
ice_sysctl_tx_itr(SYSCTL_HANDLER_ARGS)
{
        struct ice_vsi *vsi = (struct ice_vsi *)arg1;
        struct ice_softc *sc = vsi->sc;
        int increment, ret;

        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        ret = sysctl_handle_16(oidp, &vsi->tx_itr, 0, req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        /* Allow configuring a negative value to reset to the default */
        if (vsi->tx_itr < 0)
                vsi->tx_itr = ICE_DFLT_TX_ITR;
        if (vsi->tx_itr > ICE_ITR_MAX)
                vsi->tx_itr = ICE_ITR_MAX;

        /* Assume 2usec increment if it hasn't been loaded yet */
        increment = sc->hw.itr_gran ? : 2;

        /* We need to round the value to the hardware's ITR granularity */
        vsi->tx_itr = (vsi->tx_itr / increment ) * increment;

        /* If the driver has finished initializing, then we need to reprogram
         * the ITR registers now. Otherwise, they will be programmed during
         * driver initialization.
         */
        if (ice_test_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED))
                ice_configure_tx_itr(vsi);

        return (0);
}

/**
 * ice_add_vsi_tunables - Add tunables and nodes for a VSI
 * @vsi: pointer to VSI structure
 * @parent: parent node to add the tunables under
 *
 * Create a sysctl context for the VSI, so that sysctls for the VSI can be
 * dynamically removed upon VSI removal.
 *
 * Add various tunables and set up the basic node structure for the VSI. Must
 * be called *prior* to ice_add_vsi_sysctls. It should be called as soon as
 * possible after the VSI memory is initialized.
 *
 * VSI specific sysctls with CTLFLAG_TUN should be initialized here so that
 * their values can be read from loader.conf prior to their first use in the
 * driver.
 */
void
ice_add_vsi_tunables(struct ice_vsi *vsi, struct sysctl_oid *parent)
{
        struct sysctl_oid_list *vsi_list;
        char vsi_name[32], vsi_desc[32];

        struct sysctl_oid_list *parent_list = SYSCTL_CHILDREN(parent);

        /* Initialize the sysctl context for this VSI */
        sysctl_ctx_init(&vsi->ctx);

        /* Add a node to collect this VSI's statistics together */
        snprintf(vsi_name, sizeof(vsi_name), "%u", vsi->idx);
        snprintf(vsi_desc, sizeof(vsi_desc), "VSI %u", vsi->idx);
        vsi->vsi_node = SYSCTL_ADD_NODE(&vsi->ctx, parent_list, OID_AUTO, vsi_name,
                                        CTLFLAG_RD, NULL, vsi_desc);
        vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);

        vsi->rx_itr = ICE_DFLT_TX_ITR;
        SYSCTL_ADD_PROC(&vsi->ctx, vsi_list, OID_AUTO, "rx_itr",
                        CTLTYPE_S16 | CTLFLAG_RWTUN,
                        vsi, 0, ice_sysctl_rx_itr, "S",
                        ICE_SYSCTL_HELP_RX_ITR);

        vsi->tx_itr = ICE_DFLT_TX_ITR;
        SYSCTL_ADD_PROC(&vsi->ctx, vsi_list, OID_AUTO, "tx_itr",
                        CTLTYPE_S16 | CTLFLAG_RWTUN,
                        vsi, 0, ice_sysctl_tx_itr, "S",
                        ICE_SYSCTL_HELP_TX_ITR);
}

/**
 * ice_del_vsi_sysctl_ctx - Delete the sysctl context(s) of a VSI
 * @vsi: the VSI to remove contexts for
 *
 * Free the context for the VSI sysctls. This includes the main context, as
 * well as the per-queue sysctls.
 */
void
ice_del_vsi_sysctl_ctx(struct ice_vsi *vsi)
{
        device_t dev = vsi->sc->dev;
        int err;

        if (vsi->vsi_node) {
                err = sysctl_ctx_free(&vsi->ctx);
                if (err)
                        device_printf(dev, "failed to free VSI %d sysctl context, err %s\n",
                                      vsi->idx, ice_err_str(err));
                vsi->vsi_node = NULL;
        }
}

/**
 * ice_add_dscp2tc_map_sysctls - Add sysctl tree for DSCP to TC mapping
 * @sc: pointer to device private softc
 * @ctx: the sysctl ctx to use
 * @ctx_list: list of sysctl children for device (to add sysctl tree to)
 *
 * Add a sysctl tree for individual dscp2tc_map sysctls. Each child of this
 * node can map 8 DSCPs to TC values; there are 8 of these in turn for a total
 * of 64 DSCP to TC map values that the user can configure.
 */
void
ice_add_dscp2tc_map_sysctls(struct ice_softc *sc,
                            struct sysctl_ctx_list *ctx,
                            struct sysctl_oid_list *ctx_list)
{
        struct sysctl_oid_list *node_list;
        struct sysctl_oid *node;
        struct sbuf *namebuf, *descbuf;
        int first_dscp_val, last_dscp_val;

        node = SYSCTL_ADD_NODE(ctx, ctx_list, OID_AUTO, "dscp2tc_map", CTLFLAG_RD,
                               NULL, "Map of DSCP values to DCB TCs");
        node_list = SYSCTL_CHILDREN(node);

        namebuf = sbuf_new_auto();
        descbuf = sbuf_new_auto();
        for (int i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++) {
                sbuf_clear(namebuf);
                sbuf_clear(descbuf);

                first_dscp_val = i * 8;
                last_dscp_val = first_dscp_val + 7;

                sbuf_printf(namebuf, "%d-%d", first_dscp_val, last_dscp_val);
                sbuf_printf(descbuf, "Map DSCP values %d to %d to TCs",
                            first_dscp_val, last_dscp_val);

                sbuf_finish(namebuf);
                sbuf_finish(descbuf);

                SYSCTL_ADD_PROC(ctx, node_list,
                    OID_AUTO, sbuf_data(namebuf), CTLTYPE_STRING | CTLFLAG_RW,
                    sc, i, ice_sysctl_dscp2tc_map, "A", sbuf_data(descbuf));
        }

        sbuf_delete(namebuf);
        sbuf_delete(descbuf);
}

/**
 * ice_add_device_tunables - Add early tunable sysctls and sysctl nodes
 * @sc: device private structure
 *
 * Add per-device dynamic tunable sysctls, and setup the general sysctl trees
 * for re-use by ice_add_device_sysctls.
 *
 * In order for the sysctl fields to be initialized before use, this function
 * should be called as early as possible during attach activities.
 *
 * Any non-global sysctl marked as CTLFLAG_TUN should likely be initialized
 * here in this function, rather than later in ice_add_device_sysctls.
 *
 * To make things easier, this function is also expected to setup the various
 * sysctl nodes in addition to tunables so that other sysctls which can't be
 * initialized early can hook into the same nodes.
 */
void
ice_add_device_tunables(struct ice_softc *sc)
{
        device_t dev = sc->dev;

        struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
        struct sysctl_oid_list *ctx_list =
                SYSCTL_CHILDREN(device_get_sysctl_tree(dev));

        sc->enable_health_events = ice_enable_health_events;

        SYSCTL_ADD_BOOL(ctx, ctx_list, OID_AUTO, "enable_health_events",
                        CTLFLAG_RDTUN, &sc->enable_health_events, 0,
                        "Enable FW health event reporting for this PF");

        /* Add a node to track VSI sysctls. Keep track of the node in the
         * softc so that we can hook other sysctls into it later. This
         * includes both the VSI statistics, as well as potentially dynamic
         * VSIs in the future.
         */

        sc->vsi_sysctls = SYSCTL_ADD_NODE(ctx, ctx_list, OID_AUTO, "vsi",
                                          CTLFLAG_RD, NULL, "VSI Configuration and Statistics");

        /* Add debug tunables */
        ice_add_debug_tunables(sc);
}

/**
 * ice_sysctl_dump_mac_filters - Dump a list of all HW MAC Filters
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Callback for "mac_filters" sysctl to dump the programmed MAC filters.
 */
static int
ice_sysctl_dump_mac_filters(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_hw *hw = &sc->hw;
        struct ice_switch_info *sw = hw->switch_info;
        struct ice_fltr_mgmt_list_entry *fm_entry;
        struct ice_list_head *rule_head;
        struct ice_lock *rule_lock;
        struct ice_fltr_info *fi;
        struct sbuf *sbuf;
        int ret;

        UNREFERENCED_PARAMETER(oidp);
        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        /* Wire the old buffer so we can take a non-sleepable lock */
        ret = sysctl_wire_old_buffer(req, 0);
        if (ret)
                return (ret);

        sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);

        rule_lock = &sw->recp_list[ICE_SW_LKUP_MAC].filt_rule_lock;
        rule_head = &sw->recp_list[ICE_SW_LKUP_MAC].filt_rules;

        sbuf_printf(sbuf, "MAC Filter List");

        ice_acquire_lock(rule_lock);

        LIST_FOR_EACH_ENTRY(fm_entry, rule_head, ice_fltr_mgmt_list_entry, list_entry) {
                fi = &fm_entry->fltr_info;

                sbuf_printf(sbuf,
                            "\nmac = %6D, vsi_handle = %3d, fw_act_flag = %5s, lb_en = %1d, lan_en = %1d, fltr_act = %15s, fltr_rule_id = %d",
                            fi->l_data.mac.mac_addr, ":", fi->vsi_handle,
                            ice_fltr_flag_str(fi->flag), fi->lb_en, fi->lan_en,
                            ice_fwd_act_str(fi->fltr_act), fi->fltr_rule_id);

                /* if we have a vsi_list_info, print some information about that */
                if (fm_entry->vsi_list_info) {
                        sbuf_printf(sbuf,
                                    ", vsi_count = %3d, vsi_list_id = %3d, ref_cnt = %3d",
                                    fm_entry->vsi_count,
                                    fm_entry->vsi_list_info->vsi_list_id,
                                    fm_entry->vsi_list_info->ref_cnt);
                }
        }

        ice_release_lock(rule_lock);

        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        return (0);
}

/**
 * ice_sysctl_dump_vlan_filters - Dump a list of all HW VLAN Filters
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Callback for "vlan_filters" sysctl to dump the programmed VLAN filters.
 */
static int
ice_sysctl_dump_vlan_filters(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_hw *hw = &sc->hw;
        struct ice_switch_info *sw = hw->switch_info;
        struct ice_fltr_mgmt_list_entry *fm_entry;
        struct ice_list_head *rule_head;
        struct ice_lock *rule_lock;
        struct ice_fltr_info *fi;
        struct sbuf *sbuf;
        int ret;

        UNREFERENCED_PARAMETER(oidp);
        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        /* Wire the old buffer so we can take a non-sleepable lock */
        ret = sysctl_wire_old_buffer(req, 0);
        if (ret)
                return (ret);

        sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);

        rule_lock = &sw->recp_list[ICE_SW_LKUP_VLAN].filt_rule_lock;
        rule_head = &sw->recp_list[ICE_SW_LKUP_VLAN].filt_rules;

        sbuf_printf(sbuf, "VLAN Filter List");

        ice_acquire_lock(rule_lock);

        LIST_FOR_EACH_ENTRY(fm_entry, rule_head, ice_fltr_mgmt_list_entry, list_entry) {
                fi = &fm_entry->fltr_info;

                sbuf_printf(sbuf,
                            "\nvlan_id = %4d, vsi_handle = %3d, fw_act_flag = %5s, lb_en = %1d, lan_en = %1d, fltr_act = %15s, fltr_rule_id = %4d",
                            fi->l_data.vlan.vlan_id, fi->vsi_handle,
                            ice_fltr_flag_str(fi->flag), fi->lb_en, fi->lan_en,
                            ice_fwd_act_str(fi->fltr_act), fi->fltr_rule_id);

                /* if we have a vsi_list_info, print some information about that */
                if (fm_entry->vsi_list_info) {
                        sbuf_printf(sbuf,
                                    ", vsi_count = %3d, vsi_list_id = %3d, ref_cnt = %3d",
                                    fm_entry->vsi_count,
                                    fm_entry->vsi_list_info->vsi_list_id,
                                    fm_entry->vsi_list_info->ref_cnt);
                }
        }

        ice_release_lock(rule_lock);

        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        return (0);
}

/**
 * ice_sysctl_dump_ethertype_filters - Dump a list of all HW Ethertype filters
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Callback for "ethertype_filters" sysctl to dump the programmed Ethertype
 * filters.
 */
static int
ice_sysctl_dump_ethertype_filters(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_hw *hw = &sc->hw;
        struct ice_switch_info *sw = hw->switch_info;
        struct ice_fltr_mgmt_list_entry *fm_entry;
        struct ice_list_head *rule_head;
        struct ice_lock *rule_lock;
        struct ice_fltr_info *fi;
        struct sbuf *sbuf;
        int ret;

        UNREFERENCED_PARAMETER(oidp);
        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        /* Wire the old buffer so we can take a non-sleepable lock */
        ret = sysctl_wire_old_buffer(req, 0);
        if (ret)
                return (ret);

        sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);

        rule_lock = &sw->recp_list[ICE_SW_LKUP_ETHERTYPE].filt_rule_lock;
        rule_head = &sw->recp_list[ICE_SW_LKUP_ETHERTYPE].filt_rules;

        sbuf_printf(sbuf, "Ethertype Filter List");

        ice_acquire_lock(rule_lock);

        LIST_FOR_EACH_ENTRY(fm_entry, rule_head, ice_fltr_mgmt_list_entry, list_entry) {
                fi = &fm_entry->fltr_info;

                sbuf_printf(sbuf,
                            "\nethertype = 0x%04x, vsi_handle = %3d, fw_act_flag = %5s, lb_en = %1d, lan_en = %1d, fltr_act = %15s, fltr_rule_id = %4d",
                        fi->l_data.ethertype_mac.ethertype,
                        fi->vsi_handle, ice_fltr_flag_str(fi->flag),
                        fi->lb_en, fi->lan_en, ice_fwd_act_str(fi->fltr_act),
                        fi->fltr_rule_id);

                /* if we have a vsi_list_info, print some information about that */
                if (fm_entry->vsi_list_info) {
                        sbuf_printf(sbuf,
                                    ", vsi_count = %3d, vsi_list_id = %3d, ref_cnt = %3d",
                                    fm_entry->vsi_count,
                                    fm_entry->vsi_list_info->vsi_list_id,
                                    fm_entry->vsi_list_info->ref_cnt);
                }
        }

        ice_release_lock(rule_lock);

        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        return (0);
}

/**
 * ice_sysctl_dump_ethertype_mac_filters - Dump a list of all HW Ethertype/MAC filters
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Callback for "ethertype_mac_filters" sysctl to dump the programmed
 * Ethertype/MAC filters.
 */
static int
ice_sysctl_dump_ethertype_mac_filters(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_hw *hw = &sc->hw;
        struct ice_switch_info *sw = hw->switch_info;
        struct ice_fltr_mgmt_list_entry *fm_entry;
        struct ice_list_head *rule_head;
        struct ice_lock *rule_lock;
        struct ice_fltr_info *fi;
        struct sbuf *sbuf;
        int ret;

        UNREFERENCED_PARAMETER(oidp);
        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        /* Wire the old buffer so we can take a non-sleepable lock */
        ret = sysctl_wire_old_buffer(req, 0);
        if (ret)
                return (ret);

        sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);

        rule_lock = &sw->recp_list[ICE_SW_LKUP_ETHERTYPE_MAC].filt_rule_lock;
        rule_head = &sw->recp_list[ICE_SW_LKUP_ETHERTYPE_MAC].filt_rules;

        sbuf_printf(sbuf, "Ethertype/MAC Filter List");

        ice_acquire_lock(rule_lock);

        LIST_FOR_EACH_ENTRY(fm_entry, rule_head, ice_fltr_mgmt_list_entry, list_entry) {
                fi = &fm_entry->fltr_info;

                sbuf_printf(sbuf,
                            "\nethertype = 0x%04x, mac = %6D, vsi_handle = %3d, fw_act_flag = %5s, lb_en = %1d, lan_en = %1d, fltr_act = %15s, fltr_rule_id = %4d",
                            fi->l_data.ethertype_mac.ethertype,
                            fi->l_data.ethertype_mac.mac_addr, ":",
                            fi->vsi_handle, ice_fltr_flag_str(fi->flag),
                            fi->lb_en, fi->lan_en, ice_fwd_act_str(fi->fltr_act),
                            fi->fltr_rule_id);

                /* if we have a vsi_list_info, print some information about that */
                if (fm_entry->vsi_list_info) {
                        sbuf_printf(sbuf,
                                    ", vsi_count = %3d, vsi_list_id = %3d, ref_cnt = %3d",
                                    fm_entry->vsi_count,
                                    fm_entry->vsi_list_info->vsi_list_id,
                                    fm_entry->vsi_list_info->ref_cnt);
                }
        }

        ice_release_lock(rule_lock);

        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        return (0);
}

/**
 * ice_sysctl_dump_state_flags - Dump device driver state flags
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Callback for "state" sysctl to display currently set driver state flags.
 */
static int
ice_sysctl_dump_state_flags(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct sbuf *sbuf;
        u32 copied_state;
        unsigned int i;
        bool at_least_one = false;

        UNREFERENCED_PARAMETER(oidp);
        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        /* Make a copy of the state to ensure we display coherent values */
        copied_state = atomic_load_acq_32(&sc->state);

        sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);

        /* Add the string for each set state to the sbuf */
        for (i = 0; i < 32; i++) {
                if (copied_state & BIT(i)) {
                        const char *str = ice_state_to_str((enum ice_state)i);

                        at_least_one = true;

                        if (str)
                                sbuf_printf(sbuf, "\n%s", str);
                        else
                                sbuf_printf(sbuf, "\nBIT(%u)", i);
                }
        }

        if (!at_least_one)
                sbuf_printf(sbuf, "Nothing set");

        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        return (0);
}

#define ICE_SYSCTL_DEBUG_MASK_HELP \
"\nSelect debug statements to print to kernel message log"      \
"\nFlags:"                                                      \
"\n\t         0x1 - Function Tracing"                           \
"\n\t         0x2 - Driver Initialization"                      \
"\n\t         0x4 - Release"                                    \
"\n\t         0x8 - FW Logging"                                 \
"\n\t        0x10 - Link"                                       \
"\n\t        0x20 - PHY"                                        \
"\n\t        0x40 - Queue Context"                              \
"\n\t        0x80 - NVM"                                        \
"\n\t       0x100 - LAN"                                        \
"\n\t       0x200 - Flow"                                       \
"\n\t       0x400 - DCB"                                        \
"\n\t       0x800 - Diagnostics"                                \
"\n\t      0x1000 - Flow Director"                              \
"\n\t      0x2000 - Switch"                                     \
"\n\t      0x4000 - Scheduler"                                  \
"\n\t      0x8000 - RDMA"                                       \
"\n\t     0x10000 - DDP Package"                                \
"\n\t     0x20000 - Resources"                                  \
"\n\t     0x40000 - ACL"                                        \
"\n\t     0x80000 - PTP"                                        \
"\n\t   ..."                                                    \
"\n\t   0x1000000 - Admin Queue messages"                       \
"\n\t   0x2000000 - Admin Queue descriptors"                    \
"\n\t   0x4000000 - Admin Queue descriptor buffers"             \
"\n\t   0x8000000 - Admin Queue commands"                       \
"\n\t  0x10000000 - Parser"                                     \
"\n\t   ..."                                                    \
"\n\t  0x80000000 - (Reserved for user)"                        \
"\n\t"                                                          \
"\nUse \"sysctl -x\" to view flags properly."

/**
 * ice_add_debug_tunables - Add tunables helpful for debugging the device driver
 * @sc: device private structure
 *
 * Add sysctl tunable values related to debugging the device driver. For now,
 * this means a tunable to set the debug mask early during driver load.
 *
 * The debug node will be marked CTLFLAG_SKIP unless INVARIANTS is defined, so
 * that in normal kernel builds, these will all be hidden, but on a debug
 * kernel they will be more easily visible.
 */
static void
ice_add_debug_tunables(struct ice_softc *sc)
{
        struct sysctl_oid_list *debug_list;
        device_t dev = sc->dev;

        struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);
        struct sysctl_oid_list *ctx_list =
            SYSCTL_CHILDREN(device_get_sysctl_tree(dev));

        sc->debug_sysctls = SYSCTL_ADD_NODE(ctx, ctx_list, OID_AUTO, "debug",
                                            ICE_CTLFLAG_DEBUG | CTLFLAG_RD,
                                            NULL, "Debug Sysctls");
        debug_list = SYSCTL_CHILDREN(sc->debug_sysctls);

        SYSCTL_ADD_U64(ctx, debug_list, OID_AUTO, "debug_mask",
                       ICE_CTLFLAG_DEBUG | CTLFLAG_RWTUN,
                       &sc->hw.debug_mask, 0,
                       ICE_SYSCTL_DEBUG_MASK_HELP);

        /* Load the default value from the global sysctl first */
        sc->enable_tx_fc_filter = ice_enable_tx_fc_filter;

        SYSCTL_ADD_BOOL(ctx, debug_list, OID_AUTO, "enable_tx_fc_filter",
                        ICE_CTLFLAG_DEBUG | CTLFLAG_RDTUN,
                        &sc->enable_tx_fc_filter, 0,
                        "Drop Ethertype 0x8808 control frames originating from software on this PF");

        sc->tx_balance_en = ice_tx_balance_en;
        SYSCTL_ADD_BOOL(ctx, debug_list, OID_AUTO, "tx_balance",
                        ICE_CTLFLAG_DEBUG | CTLFLAG_RWTUN,
                        &sc->tx_balance_en, 0,
                        "Enable 5-layer scheduler topology");

        /* Load the default value from the global sysctl first */
        sc->enable_tx_lldp_filter = ice_enable_tx_lldp_filter;

        SYSCTL_ADD_BOOL(ctx, debug_list, OID_AUTO, "enable_tx_lldp_filter",
                        ICE_CTLFLAG_DEBUG | CTLFLAG_RDTUN,
                        &sc->enable_tx_lldp_filter, 0,
                        "Drop Ethertype 0x88cc LLDP frames originating from software on this PF");

        ice_add_fw_logging_tunables(sc, sc->debug_sysctls);
}

#define ICE_SYSCTL_HELP_REQUEST_RESET           \
"\nRequest the driver to initiate a reset."     \
"\n\tpfr - Initiate a PF reset"                 \
"\n\tcorer - Initiate a CORE reset"             \
"\n\tglobr - Initiate a GLOBAL reset"

/**
 * @var rl_sysctl_ticks
 * @brief timestamp for latest reset request sysctl call
 *
 * Helps rate-limit the call to the sysctl which resets the device
 */
int rl_sysctl_ticks = 0;

/**
 * ice_sysctl_request_reset - Request that the driver initiate a reset
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Callback for "request_reset" sysctl to request that the driver initiate
 * a reset. Expects to be passed one of the following strings
 *
 * "pfr" - Initiate a PF reset
 * "corer" - Initiate a CORE reset
 * "globr" - Initiate a Global reset
 */
static int
ice_sysctl_request_reset(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_hw *hw = &sc->hw;
        int status;
        enum ice_reset_req reset_type = ICE_RESET_INVAL;
        const char *reset_message;
        int ret;

        /* Buffer to store the requested reset string. Must contain enough
         * space to store the largest expected reset string, which currently
         * means 6 bytes of space.
         */
        char reset[6] = "";

        UNREFERENCED_PARAMETER(arg2);

        ret = priv_check(curthread, PRIV_DRIVER);
        if (ret)
                return (ret);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        /* Read in the requested reset type. */
        ret = sysctl_handle_string(oidp, reset, sizeof(reset), req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        if (strcmp(reset, "pfr") == 0) {
                reset_message = "Requesting a PF reset";
                reset_type = ICE_RESET_PFR;
        } else if (strcmp(reset, "corer") == 0) {
                reset_message = "Initiating a CORE reset";
                reset_type = ICE_RESET_CORER;
        } else if (strcmp(reset, "globr") == 0) {
                reset_message = "Initiating a GLOBAL reset";
                reset_type = ICE_RESET_GLOBR;
        } else if (strcmp(reset, "empr") == 0) {
                device_printf(sc->dev, "Triggering an EMP reset via software is not currently supported\n");
                return (EOPNOTSUPP);
        }

        if (reset_type == ICE_RESET_INVAL) {
                device_printf(sc->dev, "%s is not a valid reset request\n", reset);
                return (EINVAL);
        }

        /*
         * Rate-limit the frequency at which this function is called.
         * Assuming this is called successfully once, typically,
         * everything should be handled within the allotted time frame.
         * However, in the odd setup situations, we've also put in
         * guards for when the reset has finished, but we're in the
         * process of rebuilding. And instead of queueing an intent,
         * simply error out and let the caller retry, if so desired.
         */
        if (TICKS_2_MSEC(ticks - rl_sysctl_ticks) < 500) {
                device_printf(sc->dev,
                    "Call frequency too high. Operation aborted.\n");
                return (EBUSY);
        }
        rl_sysctl_ticks = ticks;

        if (TICKS_2_MSEC(ticks - sc->rebuild_ticks) < 100) {
                device_printf(sc->dev, "Device rebuilding. Operation aborted.\n");
                return (EBUSY);
        }

        if (rd32(hw, GLGEN_RSTAT) & GLGEN_RSTAT_DEVSTATE_M) {
                device_printf(sc->dev, "Device in reset. Operation aborted.\n");
                return (EBUSY);
        }

        device_printf(sc->dev, "%s\n", reset_message);

        /* Initiate the PF reset during the admin status task */
        if (reset_type == ICE_RESET_PFR) {
                ice_set_state(&sc->state, ICE_STATE_RESET_PFR_REQ);
                return (0);
        }

        /*
         * Other types of resets including CORE and GLOBAL resets trigger an
         * interrupt on all PFs. Initiate the reset now. Preparation and
         * rebuild logic will be handled by the admin status task.
         */
        status = ice_reset(hw, reset_type);

        /*
         * Resets can take a long time and we still don't want another call
         * to this function before we settle down.
         */
        rl_sysctl_ticks = ticks;

        if (status) {
                device_printf(sc->dev, "failed to initiate device reset, err %s\n",
                              ice_status_str(status));
                ice_set_state(&sc->state, ICE_STATE_RESET_FAILED);
                return (EFAULT);
        }

        return (0);
}

#define ICE_AQC_DBG_DUMP_CLUSTER_ID_INVALID     (0xFFFFFF)
#define ICE_SYSCTL_HELP_FW_DEBUG_DUMP_CLUSTER_SETTING           \
"\nSelect clusters to dump with \"dump\" sysctl"                \
"\nFlags:"                                                      \
"\n\t        0 - All clusters (default)"                        \
"\n\t      0x1 - Switch"                                        \
"\n\t      0x2 - ACL"                                           \
"\n\t      0x4 - Tx Scheduler"                                  \
"\n\t      0x8 - Profile Configuration"                         \
"\n\t     0x20 - Link"                                          \
"\n\t     0x80 - DCB"                                           \
"\n\t    0x100 - L2P"                                           \
"\n\t 0x400000 - Manageability Transactions (excluding E830)"   \
"\n"                                                            \
"\nUse \"sysctl -x\" to view flags properly."

/**
 * ice_sysctl_fw_debug_dump_cluster_setting - Set which clusters to dump
 *     from FW when FW debug dump occurs
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 */
static int
ice_sysctl_fw_debug_dump_cluster_setting(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        device_t dev = sc->dev;
        u32 clusters;
        int ret;

        UNREFERENCED_PARAMETER(arg2);

        ret = priv_check(curthread, PRIV_DRIVER);
        if (ret)
                return (ret);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        clusters = sc->fw_debug_dump_cluster_mask;

        ret = sysctl_handle_32(oidp, &clusters, 0, req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        u32 valid_cluster_mask;
        if (ice_is_e830(&sc->hw))
                valid_cluster_mask = ICE_FW_DEBUG_DUMP_VALID_CLUSTER_MASK_E830;
        else
                valid_cluster_mask = ICE_FW_DEBUG_DUMP_VALID_CLUSTER_MASK_E810;

        if (clusters & ~(valid_cluster_mask)) {
                device_printf(dev,
                    "%s: ERROR: Incorrect settings requested\n",
                    __func__);
                sc->fw_debug_dump_cluster_mask = ICE_AQC_DBG_DUMP_CLUSTER_ID_INVALID;
                return (EINVAL);
        }

        sc->fw_debug_dump_cluster_mask = clusters;

        return (0);
}

#define ICE_FW_DUMP_AQ_COUNT_LIMIT      (10000)

/**
 * ice_fw_debug_dump_print_cluster - Print formatted cluster data from FW
 * @sc: the device softc
 * @sbuf: initialized sbuf to print data to
 * @cluster_id: FW cluster ID to print data from
 *
 * Reads debug data from the specified cluster id in the FW and prints it to
 * the input sbuf. This function issues multiple AQ commands to the FW in
 * order to get all of the data in the cluster.
 *
 * @remark Only intended to be used by the sysctl handler
 * ice_sysctl_fw_debug_dump_do_dump
 */
static u16
ice_fw_debug_dump_print_cluster(struct ice_softc *sc, struct sbuf *sbuf, u16 cluster_id)
{
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        u16 data_buf_size = ICE_AQ_MAX_BUF_LEN;
        const u8 reserved_buf[8] = {};
        int status;
        int counter = 0;
        u8 *data_buf;

        /* Input parameters / loop variables */
        u16 table_id = 0;
        u32 offset = 0;

        /* Output from the Get Internal Data AQ command */
        u16 ret_buf_size = 0;
        u16 ret_next_cluster = 0;
        u16 ret_next_table = 0;
        u32 ret_next_index = 0;

        /* Other setup */
        data_buf = (u8 *)malloc(data_buf_size, M_ICE, M_NOWAIT | M_ZERO);
        if (!data_buf)
                return ret_next_cluster;

        ice_debug(hw, ICE_DBG_DIAG, "%s: dumping cluster id %d\n", __func__,
            cluster_id);

        for (;;) {
                /* Do not trust the FW behavior to be completely correct */
                if (counter++ >= ICE_FW_DUMP_AQ_COUNT_LIMIT) {
                        device_printf(dev,
                            "%s: Exceeded counter limit for cluster %d\n",
                            __func__, cluster_id);
                        break;
                }

                ice_debug(hw, ICE_DBG_DIAG, "---\n");
                ice_debug(hw, ICE_DBG_DIAG,
                    "table_id 0x%04x offset 0x%08x buf_size %d\n",
                    table_id, offset, data_buf_size);

                status = ice_aq_get_internal_data(hw, cluster_id, table_id,
                    offset, data_buf, data_buf_size, &ret_buf_size,
                    &ret_next_cluster, &ret_next_table, &ret_next_index, NULL);
                if (status) {
                        device_printf(dev,
                            "%s: ice_aq_get_internal_data in cluster %d: err %s aq_err %s\n",
                            __func__, cluster_id, ice_status_str(status),
                            ice_aq_str(hw->adminq.sq_last_status));
                        break;
                }

                ice_debug(hw, ICE_DBG_DIAG,
                    "ret_table_id 0x%04x ret_offset 0x%08x ret_buf_size %d\n",
                    ret_next_table, ret_next_index, ret_buf_size);

                /* Print cluster id */
                u32 print_cluster_id = (u32)cluster_id;
                sbuf_bcat(sbuf, &print_cluster_id, sizeof(print_cluster_id));
                /* Print table id */
                u32 print_table_id = (u32)table_id;
                sbuf_bcat(sbuf, &print_table_id, sizeof(print_table_id));
                /* Print table length */
                u32 print_table_length = (u32)ret_buf_size;
                sbuf_bcat(sbuf, &print_table_length, sizeof(print_table_length));
                /* Print current offset */
                u32 print_curr_offset = offset;
                sbuf_bcat(sbuf, &print_curr_offset, sizeof(print_curr_offset));
                /* Print reserved bytes */
                sbuf_bcat(sbuf, reserved_buf, sizeof(reserved_buf));
                /* Print data */
                sbuf_bcat(sbuf, data_buf, ret_buf_size);

                /* Adjust loop variables */
                memset(data_buf, 0, data_buf_size);
                bool same_table_next = (table_id == ret_next_table);
                bool last_table_next;
                if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_NEXT_CLUSTER_ID))
                        last_table_next =
                            (ret_next_table == 0xffff);
                else
                        last_table_next =
                            (ret_next_table == 0xff || ret_next_table == 0xffff);
                bool last_offset_next = (ret_next_index == 0xffffffff || ret_next_index == 0);

                if ((!same_table_next && !last_offset_next) ||
                    (same_table_next && last_table_next)) {
                        device_printf(dev,
                            "%s: Unexpected conditions for same_table_next(%d) last_table_next(%d) last_offset_next(%d), ending cluster (%d)\n",
                            __func__, same_table_next, last_table_next, last_offset_next, cluster_id);
                        break;
                }

                if (!same_table_next && !last_table_next && last_offset_next) {
                        /* We've hit the end of the table */
                        table_id = ret_next_table;
                        offset = 0;
                }
                else if (!same_table_next && last_table_next && last_offset_next) {
                        /* We've hit the end of the cluster */
                        break;
                }
                else if (same_table_next && !last_table_next && last_offset_next) {
                        if (cluster_id == 0x1 && table_id < 39)
                                table_id += 1;
                        else
                                break;
                }
                else { /* if (same_table_next && !last_table_next && !last_offset_next) */
                        /* More data left in the table */
                        offset = ret_next_index;
                }
        }

        free(data_buf, M_ICE);
        return ret_next_cluster;
}

/**
 * ice_fw_debug_dump_print_clusters - Print data from FW clusters to sbuf
 * @sc: the device softc
 * @sbuf: initialized sbuf to print data to
 *
 * Handles dumping all of the clusters to dump to the indicated sbuf. The
 * clusters do dump are determined by the value in the
 * fw_debug_dump_cluster_mask field in the sc argument.
 *
 * @remark Only intended to be used by the sysctl handler
 * ice_sysctl_fw_debug_dump_do_dump
 */
static void
ice_fw_debug_dump_print_clusters(struct ice_softc *sc, struct sbuf *sbuf)
{
        u16 next_cluster_id, max_cluster_id, start_cluster_id;
        u32 cluster_mask = sc->fw_debug_dump_cluster_mask;
        struct ice_hw *hw = &sc->hw;
        int bit;

        ice_debug(hw, ICE_DBG_DIAG, "%s: Debug Dump running...\n", __func__);

        if (ice_is_e830(hw)) {
                max_cluster_id = ICE_AQC_DBG_DUMP_CLUSTER_ID_QUEUE_MNG_E830;
                start_cluster_id = ICE_AQC_DBG_DUMP_CLUSTER_ID_SW_E830;
        } else {
                max_cluster_id = ICE_AQC_DBG_DUMP_CLUSTER_ID_QUEUE_MNG_E810;
                start_cluster_id = ICE_AQC_DBG_DUMP_CLUSTER_ID_SW_E810;
        }

        if (cluster_mask != 0) {
                for_each_set_bit(bit, &cluster_mask,
                    sizeof(cluster_mask) * BITS_PER_BYTE) {
                        ice_fw_debug_dump_print_cluster(sc, sbuf,
                            bit + start_cluster_id);
                }
        } else {
                next_cluster_id = start_cluster_id;

                /* We don't support QUEUE_MNG and FULL_CSR_SPACE */
                do {
                        next_cluster_id =
                            ice_fw_debug_dump_print_cluster(sc, sbuf, next_cluster_id);
                } while ((next_cluster_id != 0) &&
                         (next_cluster_id < max_cluster_id));
        }

}

#define ICE_SYSCTL_HELP_FW_DEBUG_DUMP_DO_DUMP \
"\nWrite 1 to output a FW debug dump containing the clusters specified by the" \
"\n\"clusters\" sysctl."                                                \
"\n"                                                                    \
"\nThe \"-b\" flag must be used in order to dump this data as binary data because" \
"\nthis data is opaque and not a string."

#define ICE_FW_DUMP_BASE_TEXT_SIZE      (1024 * 1024)
#define ICE_FW_DUMP_ALL_TEXT_SIZE       (10 * 1024 * 1024)
#define ICE_FW_DUMP_CLUST0_TEXT_SIZE    (2 * 1024 * 1024)
#define ICE_FW_DUMP_CLUST1_TEXT_SIZE    (128 * 1024)
#define ICE_FW_DUMP_CLUST2_TEXT_SIZE    (2 * 1024 * 1024)

/**
 * ice_sysctl_fw_debug_dump_do_dump - Dump data from FW to sysctl output
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Sysctl handler for the debug.dump.dump sysctl. Prints out a specially-
 * formatted dump of some debug FW data intended to be processed by a special
 * Intel tool. Prints out the cluster data specified by the "clusters"
 * sysctl.
 *
 * @remark The actual AQ calls and printing are handled by a helper
 * function above.
 */
static int
ice_sysctl_fw_debug_dump_do_dump(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        device_t dev = sc->dev;
        struct sbuf *sbuf;
        int ret;

        UNREFERENCED_PARAMETER(arg2);

        ret = priv_check(curthread, PRIV_DRIVER);
        if (ret)
                return (ret);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        /* If the user hasn't written "1" to this sysctl yet: */
        if (!ice_test_state(&sc->state, ICE_STATE_DO_FW_DEBUG_DUMP)) {
                /* Avoid output on the first set of reads to this sysctl in
                 * order to prevent a null byte from being written to the
                 * end result when called via sysctl(8).
                 */
                if (req->oldptr == NULL && req->newptr == NULL) {
                        ret = SYSCTL_OUT(req, 0, 0);
                        return (ret);
                }

                char input_buf[2] = "";
                ret = sysctl_handle_string(oidp, input_buf, sizeof(input_buf), req);
                if ((ret) || (req->newptr == NULL))
                        return (ret);

                /* If we get '1', then indicate we'll do a dump in the next
                 * sysctl read call.
                 */
                if (input_buf[0] == '1') {
                        if (sc->fw_debug_dump_cluster_mask == ICE_AQC_DBG_DUMP_CLUSTER_ID_INVALID) {
                                device_printf(dev,
                                    "%s: Debug Dump failed because an invalid cluster was specified.\n",
                                    __func__);
                                return (EINVAL);
                        }

                        ice_set_state(&sc->state, ICE_STATE_DO_FW_DEBUG_DUMP);
                        return (0);
                }

                return (EINVAL);
        }

        /* --- FW debug dump state is set --- */


        /* Caller just wants the upper bound for size */
        if (req->oldptr == NULL && req->newptr == NULL) {
                size_t est_output_len = ICE_FW_DUMP_BASE_TEXT_SIZE;
                if (sc->fw_debug_dump_cluster_mask == 0)
                        est_output_len += ICE_FW_DUMP_ALL_TEXT_SIZE;
                else {
                        if (sc->fw_debug_dump_cluster_mask & 0x1)
                                est_output_len += ICE_FW_DUMP_CLUST0_TEXT_SIZE;
                        if (sc->fw_debug_dump_cluster_mask & 0x2)
                                est_output_len += ICE_FW_DUMP_CLUST1_TEXT_SIZE;
                        if (sc->fw_debug_dump_cluster_mask & 0x4)
                                est_output_len += ICE_FW_DUMP_CLUST2_TEXT_SIZE;
                }

                ret = SYSCTL_OUT(req, 0, est_output_len);
                return (ret);
        }

        sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
        sbuf_clear_flags(sbuf, SBUF_INCLUDENUL);

        ice_fw_debug_dump_print_clusters(sc, sbuf);

        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        ice_clear_state(&sc->state, ICE_STATE_DO_FW_DEBUG_DUMP);
        return (ret);
}

/**
 * ice_add_debug_sysctls - Add sysctls helpful for debugging the device driver
 * @sc: device private structure
 *
 * Add sysctls related to debugging the device driver. Generally these should
 * simply be sysctls which dump internal driver state, to aid in understanding
 * what the driver is doing.
 */
static void
ice_add_debug_sysctls(struct ice_softc *sc)
{
        struct sysctl_oid *sw_node, *dump_node;
        struct sysctl_oid_list *debug_list, *sw_list, *dump_list;
        device_t dev = sc->dev;

        struct sysctl_ctx_list *ctx = device_get_sysctl_ctx(dev);

        debug_list = SYSCTL_CHILDREN(sc->debug_sysctls);

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "request_reset",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_STRING | CTLFLAG_WR, sc, 0,
                        ice_sysctl_request_reset, "A",
                        ICE_SYSCTL_HELP_REQUEST_RESET);

        SYSCTL_ADD_U32(ctx, debug_list, OID_AUTO, "pfr_count",
                       ICE_CTLFLAG_DEBUG | CTLFLAG_RD,
                       &sc->soft_stats.pfr_count, 0,
                       "# of PF resets handled");

        SYSCTL_ADD_U32(ctx, debug_list, OID_AUTO, "corer_count",
                       ICE_CTLFLAG_DEBUG | CTLFLAG_RD,
                       &sc->soft_stats.corer_count, 0,
                       "# of CORE resets handled");

        SYSCTL_ADD_U32(ctx, debug_list, OID_AUTO, "globr_count",
                       ICE_CTLFLAG_DEBUG | CTLFLAG_RD,
                       &sc->soft_stats.globr_count, 0,
                       "# of Global resets handled");

        SYSCTL_ADD_U32(ctx, debug_list, OID_AUTO, "empr_count",
                       ICE_CTLFLAG_DEBUG | CTLFLAG_RD,
                       &sc->soft_stats.empr_count, 0,
                       "# of EMP resets handled");

        SYSCTL_ADD_U32(ctx, debug_list, OID_AUTO, "tx_mdd_count",
                       ICE_CTLFLAG_DEBUG | CTLFLAG_RD,
                       &sc->soft_stats.tx_mdd_count, 0,
                       "# of Tx MDD events detected");

        SYSCTL_ADD_U32(ctx, debug_list, OID_AUTO, "rx_mdd_count",
                       ICE_CTLFLAG_DEBUG | CTLFLAG_RD,
                       &sc->soft_stats.rx_mdd_count, 0,
                       "# of Rx MDD events detected");

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "state",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
                        ice_sysctl_dump_state_flags, "A",
                        "Driver State Flags");

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "set_link",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_U8 | CTLFLAG_RW, sc, 0,
                        ice_sysctl_debug_set_link, "CU", "Set link");

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "phy_type_low",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_U64 | CTLFLAG_RW, sc, 0,
                        ice_sysctl_phy_type_low, "QU",
                        "PHY type Low from Get PHY Caps/Set PHY Cfg");

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "phy_type_high",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_U64 | CTLFLAG_RW, sc, 0,
                        ice_sysctl_phy_type_high, "QU",
                        "PHY type High from Get PHY Caps/Set PHY Cfg");

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "phy_sw_caps",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_STRUCT | CTLFLAG_RD, sc, 0,
                        ice_sysctl_phy_sw_caps, "",
                        "Get PHY Capabilities (Software configuration)");

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "phy_nvm_caps",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_STRUCT | CTLFLAG_RD, sc, 0,
                        ice_sysctl_phy_nvm_caps, "",
                        "Get PHY Capabilities (NVM configuration)");

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "phy_topo_caps",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_STRUCT | CTLFLAG_RD, sc, 0,
                        ice_sysctl_phy_topo_caps, "",
                        "Get PHY Capabilities (Topology configuration)");

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "phy_link_status",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_STRUCT | CTLFLAG_RD, sc, 0,
                        ice_sysctl_phy_link_status, "",
                        "Get PHY Link Status");

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "read_i2c_diag_data",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
                        ice_sysctl_read_i2c_diag_data, "A",
                        "Dump selected diagnostic data from FW");

        SYSCTL_ADD_U32(ctx, debug_list, OID_AUTO, "fw_build",
                       ICE_CTLFLAG_DEBUG | CTLFLAG_RD, &sc->hw.fw_build, 0,
                       "FW Build ID");

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "os_ddp_version",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
                        ice_sysctl_os_pkg_version, "A",
                        "DDP package name and version found in ice_ddp");

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "cur_lldp_persist_status",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
                        ice_sysctl_fw_cur_lldp_persist_status, "A",
                        "Current LLDP persistent status");

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "dflt_lldp_persist_status",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
                        ice_sysctl_fw_dflt_lldp_persist_status, "A",
                        "Default LLDP persistent status");

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "negotiated_fc",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
                        ice_sysctl_negotiated_fc, "A",
                        "Current Negotiated Flow Control mode");

        if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_PHY_STATISTICS)) {
                SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "phy_statistics",
                                CTLTYPE_STRING | CTLFLAG_RD,
                                sc, 0, ice_sysctl_dump_phy_stats, "A",
                                "Dumps PHY statistics from firmware");
        }

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "local_dcbx_cfg",
                        CTLTYPE_STRING | CTLFLAG_RD, sc, ICE_AQ_LLDP_MIB_LOCAL,
                        ice_sysctl_dump_dcbx_cfg, "A",
                        "Dumps Local MIB information from firmware");

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "remote_dcbx_cfg",
                        CTLTYPE_STRING | CTLFLAG_RD, sc, ICE_AQ_LLDP_MIB_REMOTE,
                        ice_sysctl_dump_dcbx_cfg, "A",
                        "Dumps Remote MIB information from firmware");

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "pf_vsi_cfg", CTLTYPE_STRING | CTLFLAG_RD,
                        sc, 0, ice_sysctl_dump_vsi_cfg, "A",
                        "Dumps Selected PF VSI parameters from firmware");

        SYSCTL_ADD_PROC(ctx, debug_list, OID_AUTO, "query_port_ets", CTLTYPE_STRING | CTLFLAG_RD,
                        sc, 0, ice_sysctl_query_port_ets, "A",
                        "Prints selected output from Query Port ETS AQ command");

        SYSCTL_ADD_U64(ctx, debug_list, OID_AUTO, "rx_length_errors",
                       CTLFLAG_RD | CTLFLAG_STATS, &sc->stats.cur.rx_len_errors, 0,
                       "Receive Length Errors (SNAP packets)");

        sw_node = SYSCTL_ADD_NODE(ctx, debug_list, OID_AUTO, "switch",
                                  ICE_CTLFLAG_DEBUG | CTLFLAG_RD, NULL,
                                  "Switch Configuration");
        sw_list = SYSCTL_CHILDREN(sw_node);

        SYSCTL_ADD_PROC(ctx, sw_list, OID_AUTO, "mac_filters",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
                        ice_sysctl_dump_mac_filters, "A",
                        "MAC Filters");

        SYSCTL_ADD_PROC(ctx, sw_list, OID_AUTO, "vlan_filters",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
                        ice_sysctl_dump_vlan_filters, "A",
                        "VLAN Filters");

        SYSCTL_ADD_PROC(ctx, sw_list, OID_AUTO, "ethertype_filters",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
                        ice_sysctl_dump_ethertype_filters, "A",
                        "Ethertype Filters");

        SYSCTL_ADD_PROC(ctx, sw_list, OID_AUTO, "ethertype_mac_filters",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_STRING | CTLFLAG_RD, sc, 0,
                        ice_sysctl_dump_ethertype_mac_filters, "A",
                        "Ethertype/MAC Filters");

        dump_node = SYSCTL_ADD_NODE(ctx, debug_list, OID_AUTO, "dump",
                                  ICE_CTLFLAG_DEBUG | CTLFLAG_RD, NULL,
                                  "Internal FW Dump");
        dump_list = SYSCTL_CHILDREN(dump_node);

        SYSCTL_ADD_PROC(ctx, dump_list, OID_AUTO, "clusters",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_U32 | CTLFLAG_RW, sc, 0,
                        ice_sysctl_fw_debug_dump_cluster_setting, "SU",
                        ICE_SYSCTL_HELP_FW_DEBUG_DUMP_CLUSTER_SETTING);

        SYSCTL_ADD_PROC(ctx, dump_list, OID_AUTO, "dump",
                        ICE_CTLFLAG_DEBUG | CTLTYPE_STRING | CTLFLAG_RW | CTLFLAG_MPSAFE, sc, 0,
                        ice_sysctl_fw_debug_dump_do_dump, "",
                        ICE_SYSCTL_HELP_FW_DEBUG_DUMP_DO_DUMP);
}

/**
 * ice_vsi_disable_tx - Disable (unconfigure) Tx queues for a VSI
 * @vsi: the VSI to disable
 *
 * Disables the Tx queues associated with this VSI. Essentially the opposite
 * of ice_cfg_vsi_for_tx.
 */
int
ice_vsi_disable_tx(struct ice_vsi *vsi)
{
        struct ice_softc *sc = vsi->sc;
        struct ice_hw *hw = &sc->hw;
        int status;
        u32 *q_teids;
        u16 *q_ids, *q_handles;
        size_t q_teids_size, q_ids_size, q_handles_size;
        int tc, j, buf_idx, err = 0;

        if (vsi->num_tx_queues > 255)
                return (ENOSYS);

        q_teids_size = sizeof(*q_teids) * vsi->num_tx_queues;
        q_teids = (u32 *)malloc(q_teids_size, M_ICE, M_NOWAIT|M_ZERO);
        if (!q_teids)
                return (ENOMEM);

        q_ids_size = sizeof(*q_ids) * vsi->num_tx_queues;
        q_ids = (u16 *)malloc(q_ids_size, M_ICE, M_NOWAIT|M_ZERO);
        if (!q_ids) {
                err = (ENOMEM);
                goto free_q_teids;
        }

        q_handles_size = sizeof(*q_handles) * vsi->num_tx_queues;
        q_handles = (u16 *)malloc(q_handles_size, M_ICE, M_NOWAIT|M_ZERO);
        if (!q_handles) {
                err = (ENOMEM);
                goto free_q_ids;
        }

        ice_for_each_traffic_class(tc) {
                struct ice_tc_info *tc_info = &vsi->tc_info[tc];
                u16 start_idx, end_idx;

                /* Skip rest of disabled TCs once the first
                 * disabled TC is found */
                if (!(vsi->tc_map & BIT(tc)))
                        break;

                /* Fill out TX queue information for this TC */
                start_idx = tc_info->qoffset;
                end_idx = start_idx + tc_info->qcount_tx;
                buf_idx = 0;
                for (j = start_idx; j < end_idx; j++) {
                        struct ice_tx_queue *txq = &vsi->tx_queues[j];

                        q_ids[buf_idx] = vsi->tx_qmap[j];
                        q_handles[buf_idx] = txq->q_handle;
                        q_teids[buf_idx] = txq->q_teid;
                        buf_idx++;
                }

                status = ice_dis_vsi_txq(hw->port_info, vsi->idx, tc, buf_idx,
                                         q_handles, q_ids, q_teids, ICE_NO_RESET, 0, NULL);
                if (status == ICE_ERR_DOES_NOT_EXIST) {
                        ; /* Queues have already been disabled, no need to report this as an error */
                } else if (status == ICE_ERR_RESET_ONGOING) {
                        device_printf(sc->dev,
                                      "Reset in progress. LAN Tx queues already disabled\n");
                        break;
                } else if (status) {
                        device_printf(sc->dev,
                                      "Failed to disable LAN Tx queues: err %s aq_err %s\n",
                                      ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
                        err = (ENODEV);
                        break;
                }

                /* Clear buffers */
                memset(q_teids, 0, q_teids_size);
                memset(q_ids, 0, q_ids_size);
                memset(q_handles, 0, q_handles_size);
        }

/* free_q_handles: */
        free(q_handles, M_ICE);
free_q_ids:
        free(q_ids, M_ICE);
free_q_teids:
        free(q_teids, M_ICE);

        return err;
}

/**
 * ice_vsi_set_rss_params - Set the RSS parameters for the VSI
 * @vsi: the VSI to configure
 *
 * Sets the RSS table size and lookup table type for the VSI based on its
 * VSI type.
 */
static void
ice_vsi_set_rss_params(struct ice_vsi *vsi)
{
        struct ice_softc *sc = vsi->sc;
        struct ice_hw_common_caps *cap;

        cap = &sc->hw.func_caps.common_cap;

        switch (vsi->type) {
        case ICE_VSI_PF:
                /* The PF VSI inherits RSS instance of the PF */
                vsi->rss_table_size = cap->rss_table_size;
                vsi->rss_lut_type = ICE_LUT_PF;
                break;
        case ICE_VSI_VF:
        case ICE_VSI_VMDQ2:
                vsi->rss_table_size = ICE_VSIQF_HLUT_ARRAY_SIZE;
                vsi->rss_lut_type = ICE_LUT_VSI;
                break;
        default:
                device_printf(sc->dev,
                              "VSI %d: RSS not supported for VSI type %d\n",
                              vsi->idx, vsi->type);
                break;
        }
}

/**
 * ice_vsi_add_txqs_ctx - Create a sysctl context and node to store txq sysctls
 * @vsi: The VSI to add the context for
 *
 * Creates a sysctl context for storing txq sysctls. Additionally creates
 * a node rooted at the given VSI's main sysctl node. This context will be
 * used to store per-txq sysctls which may need to be released during the
 * driver's lifetime.
 */
void
ice_vsi_add_txqs_ctx(struct ice_vsi *vsi)
{
        struct sysctl_oid_list *vsi_list;

        sysctl_ctx_init(&vsi->txqs_ctx);

        vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);

        vsi->txqs_node = SYSCTL_ADD_NODE(&vsi->txqs_ctx, vsi_list, OID_AUTO, "txqs",
                                         CTLFLAG_RD, NULL, "Tx Queues");
}

/**
 * ice_vsi_add_rxqs_ctx - Create a sysctl context and node to store rxq sysctls
 * @vsi: The VSI to add the context for
 *
 * Creates a sysctl context for storing rxq sysctls. Additionally creates
 * a node rooted at the given VSI's main sysctl node. This context will be
 * used to store per-rxq sysctls which may need to be released during the
 * driver's lifetime.
 */
void
ice_vsi_add_rxqs_ctx(struct ice_vsi *vsi)
{
        struct sysctl_oid_list *vsi_list;

        sysctl_ctx_init(&vsi->rxqs_ctx);

        vsi_list = SYSCTL_CHILDREN(vsi->vsi_node);

        vsi->rxqs_node = SYSCTL_ADD_NODE(&vsi->rxqs_ctx, vsi_list, OID_AUTO, "rxqs",
                                         CTLFLAG_RD, NULL, "Rx Queues");
}

/**
 * ice_vsi_del_txqs_ctx - Delete the Tx queue sysctl context for this VSI
 * @vsi: The VSI to delete from
 *
 * Frees the txq sysctl context created for storing the per-queue Tx sysctls.
 * Must be called prior to freeing the Tx queue memory, in order to avoid
 * having sysctls point at stale memory.
 */
void
ice_vsi_del_txqs_ctx(struct ice_vsi *vsi)
{
        device_t dev = vsi->sc->dev;
        int err;

        if (vsi->txqs_node) {
                err = sysctl_ctx_free(&vsi->txqs_ctx);
                if (err)
                        device_printf(dev, "failed to free VSI %d txqs_ctx, err %s\n",
                                      vsi->idx, ice_err_str(err));
                vsi->txqs_node = NULL;
        }
}

/**
 * ice_vsi_del_rxqs_ctx - Delete the Rx queue sysctl context for this VSI
 * @vsi: The VSI to delete from
 *
 * Frees the rxq sysctl context created for storing the per-queue Rx sysctls.
 * Must be called prior to freeing the Rx queue memory, in order to avoid
 * having sysctls point at stale memory.
 */
void
ice_vsi_del_rxqs_ctx(struct ice_vsi *vsi)
{
        device_t dev = vsi->sc->dev;
        int err;

        if (vsi->rxqs_node) {
                err = sysctl_ctx_free(&vsi->rxqs_ctx);
                if (err)
                        device_printf(dev, "failed to free VSI %d rxqs_ctx, err %s\n",
                                      vsi->idx, ice_err_str(err));
                vsi->rxqs_node = NULL;
        }
}

/**
 * ice_add_txq_sysctls - Add per-queue sysctls for a Tx queue
 * @txq: pointer to the Tx queue
 *
* Add per-queue sysctls for a given Tx queue. Can't be called during
* ice_add_vsi_sysctls, since the queue memory has not yet been setup.
 */
void
ice_add_txq_sysctls(struct ice_tx_queue *txq)
{
        struct ice_vsi *vsi = txq->vsi;
        struct sysctl_ctx_list *ctx = &vsi->txqs_ctx;
        struct sysctl_oid_list *txqs_list, *this_txq_list;
        struct sysctl_oid *txq_node;
        char txq_name[32], txq_desc[32];

        const struct ice_sysctl_info ctls[] = {
                { &txq->stats.tx_packets, "tx_packets", "Queue Packets Transmitted" },
                { &txq->stats.tx_bytes, "tx_bytes", "Queue Bytes Transmitted" },
                { &txq->stats.mss_too_small, "mss_too_small", "TSO sends with an MSS less than 64" },
                { &txq->stats.tso, "tso", "TSO packets" },
                { 0, 0, 0 }
        };

        const struct ice_sysctl_info *entry = ctls;

        txqs_list = SYSCTL_CHILDREN(vsi->txqs_node);

        snprintf(txq_name, sizeof(txq_name), "%u", txq->me);
        snprintf(txq_desc, sizeof(txq_desc), "Tx Queue %u", txq->me);
        txq_node = SYSCTL_ADD_NODE(ctx, txqs_list, OID_AUTO, txq_name,
                                   CTLFLAG_RD, NULL, txq_desc);
        this_txq_list = SYSCTL_CHILDREN(txq_node);

        /* Add the Tx queue statistics */
        while (entry->stat != 0) {
                SYSCTL_ADD_U64(ctx, this_txq_list, OID_AUTO, entry->name,
                               CTLFLAG_RD | CTLFLAG_STATS, entry->stat, 0,
                               entry->description);
                entry++;
        }

        SYSCTL_ADD_U8(ctx, this_txq_list, OID_AUTO, "tc",
                       CTLFLAG_RD, &txq->tc, 0,
                       "Traffic Class that Queue belongs to");
}

/**
 * ice_add_rxq_sysctls - Add per-queue sysctls for an Rx queue
 * @rxq: pointer to the Rx queue
 *
 * Add per-queue sysctls for a given Rx queue. Can't be called during
 * ice_add_vsi_sysctls, since the queue memory has not yet been setup.
 */
void
ice_add_rxq_sysctls(struct ice_rx_queue *rxq)
{
        struct ice_vsi *vsi = rxq->vsi;
        struct sysctl_ctx_list *ctx = &vsi->rxqs_ctx;
        struct sysctl_oid_list *rxqs_list, *this_rxq_list;
        struct sysctl_oid *rxq_node;
        char rxq_name[32], rxq_desc[32];

        const struct ice_sysctl_info ctls[] = {
                { &rxq->stats.rx_packets, "rx_packets", "Queue Packets Received" },
                { &rxq->stats.rx_bytes, "rx_bytes", "Queue Bytes Received" },
                { &rxq->stats.desc_errs, "rx_desc_errs", "Queue Rx Descriptor Errors" },
                { 0, 0, 0 }
        };

        const struct ice_sysctl_info *entry = ctls;

        rxqs_list = SYSCTL_CHILDREN(vsi->rxqs_node);

        snprintf(rxq_name, sizeof(rxq_name), "%u", rxq->me);
        snprintf(rxq_desc, sizeof(rxq_desc), "Rx Queue %u", rxq->me);
        rxq_node = SYSCTL_ADD_NODE(ctx, rxqs_list, OID_AUTO, rxq_name,
                                   CTLFLAG_RD, NULL, rxq_desc);
        this_rxq_list = SYSCTL_CHILDREN(rxq_node);

        /* Add the Rx queue statistics */
        while (entry->stat != 0) {
                SYSCTL_ADD_U64(ctx, this_rxq_list, OID_AUTO, entry->name,
                               CTLFLAG_RD | CTLFLAG_STATS, entry->stat, 0,
                               entry->description);
                entry++;
        }

        SYSCTL_ADD_U8(ctx, this_rxq_list, OID_AUTO, "tc",
                       CTLFLAG_RD, &rxq->tc, 0,
                       "Traffic Class that Queue belongs to");
}

/**
 * ice_set_rss_key - Configure a given VSI with the default RSS key
 * @vsi: the VSI to configure
 *
 * Program the hardware RSS key. We use rss_getkey to grab the kernel RSS key.
 */
static int
ice_set_rss_key(struct ice_vsi *vsi)
{
        struct ice_aqc_get_set_rss_keys keydata = { .standard_rss_key = {0} };
        struct ice_softc *sc = vsi->sc;
        struct ice_hw *hw = &sc->hw;
        int status;

        /*
         * Even if the RSS kernel interface is disabled, this function
         * is still available.
         */
        rss_getkey(keydata.standard_rss_key);

        status = ice_aq_set_rss_key(hw, vsi->idx, &keydata);
        if (status) {
                device_printf(sc->dev,
                    "ice_aq_set_rss_key status %s, error %s\n",
                    ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        return (0);
}

/**
 * ice_set_rss_flow_flds - Program the RSS hash flows after package init
 * @vsi: the VSI to configure
 *
 * If the package file is initialized, the default RSS flows are reset. We
 * need to reprogram the expected hash configuration. We'll use
 * rss_gethashconfig() to determine which flows to enable. If RSS kernel
 * support is not enabled, this macro will fall back to suitable defaults.
 */
static void
ice_set_rss_flow_flds(struct ice_vsi *vsi)
{
        struct ice_softc *sc = vsi->sc;
        struct ice_hw *hw = &sc->hw;
        struct ice_rss_hash_cfg rss_cfg = { 0, 0, ICE_RSS_ANY_HEADERS, false };
        device_t dev = sc->dev;
        int status;
        u_int rss_hash_config;

        rss_hash_config = rss_gethashconfig();

        if (rss_hash_config & RSS_HASHTYPE_RSS_IPV4) {
                rss_cfg.addl_hdrs = ICE_FLOW_SEG_HDR_IPV4;
                rss_cfg.hash_flds = ICE_FLOW_HASH_IPV4;
                status = ice_add_rss_cfg(hw, vsi->idx, &rss_cfg);
                if (status)
                        device_printf(dev,
                                      "ice_add_rss_cfg on VSI %d failed for ipv4 flow, err %s aq_err %s\n",
                                      vsi->idx, ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
        }
        if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV4) {
                rss_cfg.addl_hdrs = ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_TCP;
                rss_cfg.hash_flds = ICE_HASH_TCP_IPV4;
                status = ice_add_rss_cfg(hw, vsi->idx, &rss_cfg);
                if (status)
                        device_printf(dev,
                                      "ice_add_rss_cfg on VSI %d failed for tcp4 flow, err %s aq_err %s\n",
                                      vsi->idx, ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
        }
        if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV4) {
                rss_cfg.addl_hdrs = ICE_FLOW_SEG_HDR_IPV4 | ICE_FLOW_SEG_HDR_UDP;
                rss_cfg.hash_flds = ICE_HASH_UDP_IPV4;
                status = ice_add_rss_cfg(hw, vsi->idx, &rss_cfg);
                if (status)
                        device_printf(dev,
                                      "ice_add_rss_cfg on VSI %d failed for udp4 flow, err %s aq_err %s\n",
                                      vsi->idx, ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
        }
        if (rss_hash_config & (RSS_HASHTYPE_RSS_IPV6 | RSS_HASHTYPE_RSS_IPV6_EX)) {
                rss_cfg.addl_hdrs = ICE_FLOW_SEG_HDR_IPV6;
                rss_cfg.hash_flds = ICE_FLOW_HASH_IPV6;
                status = ice_add_rss_cfg(hw, vsi->idx, &rss_cfg);
                if (status)
                        device_printf(dev,
                                      "ice_add_rss_cfg on VSI %d failed for ipv6 flow, err %s aq_err %s\n",
                                      vsi->idx, ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
        }
        if (rss_hash_config & RSS_HASHTYPE_RSS_TCP_IPV6) {
                rss_cfg.addl_hdrs = ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_TCP;
                rss_cfg.hash_flds = ICE_HASH_TCP_IPV6;
                status = ice_add_rss_cfg(hw, vsi->idx, &rss_cfg);
                if (status)
                        device_printf(dev,
                                      "ice_add_rss_cfg on VSI %d failed for tcp6 flow, err %s aq_err %s\n",
                                      vsi->idx, ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
        }
        if (rss_hash_config & RSS_HASHTYPE_RSS_UDP_IPV6) {
                rss_cfg.addl_hdrs = ICE_FLOW_SEG_HDR_IPV6 | ICE_FLOW_SEG_HDR_UDP;
                rss_cfg.hash_flds = ICE_HASH_UDP_IPV6;
                status = ice_add_rss_cfg(hw, vsi->idx, &rss_cfg);
                if (status)
                        device_printf(dev,
                                      "ice_add_rss_cfg on VSI %d failed for udp6 flow, err %s aq_err %s\n",
                                      vsi->idx, ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
        }

        /* Warn about RSS hash types which are not supported */
        /* coverity[dead_error_condition] */
        if (rss_hash_config & ~ICE_DEFAULT_RSS_HASH_CONFIG) {
                device_printf(dev,
                              "ice_add_rss_cfg on VSI %d could not configure every requested hash type\n",
                              vsi->idx);
        }
}

/**
 * ice_set_rss_lut - Program the RSS lookup table for a VSI
 * @vsi: the VSI to configure
 *
 * Programs the RSS lookup table for a given VSI. We use
 * rss_get_indirection_to_bucket which will use the indirection table provided
 * by the kernel RSS interface when available. If the kernel RSS interface is
 * not available, we will fall back to a simple round-robin fashion queue
 * assignment.
 */
static int
ice_set_rss_lut(struct ice_vsi *vsi)
{
        struct ice_softc *sc = vsi->sc;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        struct ice_aq_get_set_rss_lut_params lut_params;
        int status;
        int i, err = 0;
        u8 *lut;

        lut = (u8 *)malloc(vsi->rss_table_size, M_ICE, M_NOWAIT|M_ZERO);
        if (!lut) {
                device_printf(dev, "Failed to allocate RSS lut memory\n");
                return (ENOMEM);
        }

        /* Populate the LUT with max no. of queues. If the RSS kernel
         * interface is disabled, this will assign the lookup table in
         * a simple round robin fashion
         */
        for (i = 0; i < vsi->rss_table_size; i++) {
                /* XXX: this needs to be changed if num_rx_queues ever counts
                 * more than just the RSS queues */
                lut[i] = rss_get_indirection_to_bucket(i) % vsi->num_rx_queues;
        }

        lut_params.vsi_handle = vsi->idx;
        lut_params.lut_size = vsi->rss_table_size;
        lut_params.lut_type = vsi->rss_lut_type;
        lut_params.lut = lut;
        lut_params.global_lut_id = 0;
        status = ice_aq_set_rss_lut(hw, &lut_params);
        if (status) {
                device_printf(dev,
                              "Cannot set RSS lut, err %s aq_err %s\n",
                              ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
                err = (EIO);
        }

        free(lut, M_ICE);
        return err;
}

/**
 * ice_config_rss - Configure RSS for a VSI
 * @vsi: the VSI to configure
 *
 * If FEATURE_RSS is enabled, configures the RSS lookup table and hash key for
 * a given VSI.
 */
int
ice_config_rss(struct ice_vsi *vsi)
{
        int err;

        /* Nothing to do, if RSS is not enabled */
        if (!ice_is_bit_set(vsi->sc->feat_en, ICE_FEATURE_RSS))
                return 0;

        err = ice_set_rss_key(vsi);
        if (err)
                return err;

        ice_set_rss_flow_flds(vsi);

        return ice_set_rss_lut(vsi);
}

/**
 * ice_log_pkg_init - Log a message about status of DDP initialization
 * @sc: the device softc pointer
 * @pkg_status: the status result of ice_copy_and_init_pkg
 *
 * Called by ice_load_pkg after an attempt to download the DDP package
 * contents to the device to log an appropriate message for the system
 * administrator about download status.
 *
 * @post ice_is_init_pkg_successful function is used to determine
 * whether the download was successful and DDP package is compatible
 * with this driver. Otherwise driver will transition to Safe Mode.
 */
void
ice_log_pkg_init(struct ice_softc *sc, enum ice_ddp_state pkg_status)
{
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        struct sbuf *active_pkg, *os_pkg;

        active_pkg = sbuf_new_auto();
        ice_active_pkg_version_str(hw, active_pkg);
        sbuf_finish(active_pkg);

        os_pkg = sbuf_new_auto();
        ice_os_pkg_version_str(hw, os_pkg);
        sbuf_finish(os_pkg);

        switch (pkg_status) {
        case ICE_DDP_PKG_SUCCESS:
                device_printf(dev,
                              "The DDP package was successfully loaded: %s.\n",
                              sbuf_data(active_pkg));
                break;
        case ICE_DDP_PKG_SAME_VERSION_ALREADY_LOADED:
        case ICE_DDP_PKG_ALREADY_LOADED:
                device_printf(dev,
                              "DDP package already present on device: %s.\n",
                              sbuf_data(active_pkg));
                break;
        case ICE_DDP_PKG_COMPATIBLE_ALREADY_LOADED:
                device_printf(dev,
                              "The driver could not load the DDP package file because a compatible DDP package is already present on the device.  The device has package %s.  The ice_ddp module has package: %s.\n",
                              sbuf_data(active_pkg),
                              sbuf_data(os_pkg));
                break;
        case ICE_DDP_PKG_FILE_VERSION_TOO_HIGH:
                device_printf(dev,
                              "The device has a DDP package that is higher than the driver supports.  The device has package %s.  The driver requires version %d.%d.x.x.  Entering Safe Mode.\n",
                              sbuf_data(active_pkg),
                              ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
                break;
        case ICE_DDP_PKG_FILE_VERSION_TOO_LOW:
                device_printf(dev,
                              "The device has a DDP package that is lower than the driver supports.  The device has package %s.  The driver requires version %d.%d.x.x.  Entering Safe Mode.\n",
                              sbuf_data(active_pkg),
                              ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
                break;
        case ICE_DDP_PKG_ALREADY_LOADED_NOT_SUPPORTED:
                /*
                 * This assumes that the active_pkg_ver will not be
                 * initialized if the ice_ddp package version is not
                 * supported.
                 */
                if (pkg_ver_empty(&hw->active_pkg_ver, hw->active_pkg_name)) {
                        /* The ice_ddp version is not supported */
                        if (pkg_ver_compatible(&hw->pkg_ver) > 0) {
                                device_printf(dev,
                                              "The DDP package in the ice_ddp module is higher than the driver supports.  The ice_ddp module has package %s.  The driver requires version %d.%d.x.x.  Please use an updated driver.  Entering Safe Mode.\n",
                                              sbuf_data(os_pkg),
                                              ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
                        } else if (pkg_ver_compatible(&hw->pkg_ver) < 0) {
                                device_printf(dev,
                                              "The DDP package in the ice_ddp module is lower than the driver supports.  The ice_ddp module has package %s.  The driver requires version %d.%d.x.x.  Please use an updated ice_ddp module.  Entering Safe Mode.\n",
                                              sbuf_data(os_pkg),
                                              ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
                        } else {
                                device_printf(dev,
                                              "An unknown error occurred when loading the DDP package.  The ice_ddp module has package %s.  The device has package %s.  The driver requires version %d.%d.x.x.  Entering Safe Mode.\n",
                                              sbuf_data(os_pkg),
                                              sbuf_data(active_pkg),
                                              ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
                        }
                } else {
                        if (pkg_ver_compatible(&hw->active_pkg_ver) > 0) {
                                device_printf(dev,
                                              "The device has a DDP package that is higher than the driver supports.  The device has package %s.  The driver requires version %d.%d.x.x.  Entering Safe Mode.\n",
                                              sbuf_data(active_pkg),
                                              ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
                        } else if (pkg_ver_compatible(&hw->active_pkg_ver) < 0) {
                                device_printf(dev,
                                              "The device has a DDP package that is lower than the driver supports.  The device has package %s.  The driver requires version %d.%d.x.x.  Entering Safe Mode.\n",
                                              sbuf_data(active_pkg),
                                              ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
                        } else {
                                device_printf(dev,
                                              "An unknown error occurred when loading the DDP package.  The ice_ddp module has package %s.  The device has package %s.  The driver requires version %d.%d.x.x.  Entering Safe Mode.\n",
                                              sbuf_data(os_pkg),
                                              sbuf_data(active_pkg),
                                              ICE_PKG_SUPP_VER_MAJ, ICE_PKG_SUPP_VER_MNR);
                        }
                }
                break;
        case ICE_DDP_PKG_INVALID_FILE:
                device_printf(dev,
                              "The DDP package in the ice_ddp module is invalid.  Entering Safe Mode\n");
                break;
        case ICE_DDP_PKG_FW_MISMATCH:
                device_printf(dev,
                              "The firmware loaded on the device is not compatible with the DDP package.  Please update the device's NVM.  Entering safe mode.\n");
                break;
        case ICE_DDP_PKG_NO_SEC_MANIFEST:
        case ICE_DDP_PKG_FILE_SIGNATURE_INVALID:
                device_printf(dev,
                              "The DDP package in the ice_ddp module cannot be loaded because its signature is not valid.  Please use a valid ice_ddp module.  Entering Safe Mode.\n");
                break;
        case ICE_DDP_PKG_SECURE_VERSION_NBR_TOO_LOW:
                device_printf(dev,
                              "The DDP package in the ice_ddp module could not be loaded because its security revision is too low.  Please use an updated ice_ddp module.  Entering Safe Mode.\n");
                break;
        case ICE_DDP_PKG_MANIFEST_INVALID:
        case ICE_DDP_PKG_BUFFER_INVALID:
                device_printf(dev,
                              "An error occurred on the device while loading the DDP package.  Entering Safe Mode.\n");
                break;
        default:
                device_printf(dev,
                         "An unknown error occurred when loading the DDP package.  Entering Safe Mode.\n");
                break;
        }

        sbuf_delete(active_pkg);
        sbuf_delete(os_pkg);
}

/**
 * ice_load_pkg_file - Load the DDP package file using firmware_get
 * @sc: device private softc
 *
 * Use firmware_get to load the DDP package memory and then request that
 * firmware download the package contents and program the relevant hardware
 * bits.
 *
 * This function makes a copy of the DDP package memory which is tracked in
 * the ice_hw structure. The copy will be managed and released by
 * ice_deinit_hw(). This allows the firmware reference to be immediately
 * released using firmware_put.
 */
int
ice_load_pkg_file(struct ice_softc *sc)
{
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        enum ice_ddp_state state;
        const struct firmware *pkg;
        int status = 0;
        u8 cached_layer_count;
        u8 *buf_copy;

        pkg = firmware_get("ice_ddp");
        if (!pkg) {
                device_printf(dev,
                    "The DDP package module (ice_ddp) failed to load or could not be found. Entering Safe Mode.\n");
                if (cold)
                        device_printf(dev,
                            "The DDP package module cannot be automatically loaded while booting. You may want to specify ice_ddp_load=\"YES\" in your loader.conf\n");
                status = ICE_ERR_CFG;
                goto err_load_pkg;
        }

        /* Check for topology change */
        if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_TX_BALANCE)) {
                cached_layer_count = hw->num_tx_sched_layers;
                buf_copy = (u8 *)malloc(pkg->datasize, M_ICE, M_NOWAIT);
                if (buf_copy == NULL)
                        return ICE_ERR_NO_MEMORY;
                memcpy(buf_copy, pkg->data, pkg->datasize);
                status = ice_cfg_tx_topo(&sc->hw, buf_copy, pkg->datasize);
                free(buf_copy, M_ICE);
                /* Success indicates a change was made */
                if (!status) {
                        /* 9 -> 5 */
                        if (cached_layer_count == 9)
                                device_printf(dev,
                                    "Transmit balancing feature enabled\n");
                        else
                                device_printf(dev,
                                    "Transmit balancing feature disabled\n");
                        ice_set_bit(ICE_FEATURE_TX_BALANCE, sc->feat_en);
                        return (status);
                } else if (status == ICE_ERR_CFG) {
                        /* Status is ICE_ERR_CFG when DDP does not support transmit balancing */
                        device_printf(dev,
                            "DDP package does not support transmit balancing feature - please update to the latest DDP package and try again\n");
                } else if (status == ICE_ERR_ALREADY_EXISTS) {
                        /* Requested config already loaded */
                } else if (status == ICE_ERR_AQ_ERROR) {
                        device_printf(dev,
                            "Error configuring transmit balancing: %s\n",
                            ice_status_str(status));
                }
        }

        /* Copy and download the pkg contents */
        state = ice_copy_and_init_pkg(hw, (const u8 *)pkg->data, pkg->datasize);

        /* Release the firmware reference */
        firmware_put(pkg, FIRMWARE_UNLOAD);

        /* Check the active DDP package version and log a message */
        ice_log_pkg_init(sc, state);

        /* Place the driver into safe mode */
        if (ice_is_init_pkg_successful(state))
                return (ICE_ERR_ALREADY_EXISTS);

err_load_pkg:
        ice_zero_bitmap(sc->feat_cap, ICE_FEATURE_COUNT);
        ice_zero_bitmap(sc->feat_en, ICE_FEATURE_COUNT);
        ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_cap);
        ice_set_bit(ICE_FEATURE_SAFE_MODE, sc->feat_en);

        return (status);
}

/**
 * ice_get_ifnet_counter - Retrieve counter value for a given ifnet counter
 * @vsi: the vsi to retrieve the value for
 * @counter: the counter type to retrieve
 *
 * Returns the value for a given ifnet counter. To do so, we calculate the
 * value based on the matching hardware statistics.
 */
uint64_t
ice_get_ifnet_counter(struct ice_vsi *vsi, ift_counter counter)
{
        struct ice_hw_port_stats *hs = &vsi->sc->stats.cur;
        struct ice_eth_stats *es = &vsi->hw_stats.cur;

        /* For some statistics, especially those related to error flows, we do
         * not have per-VSI counters. In this case, we just report the global
         * counters.
         */

        switch (counter) {
        case IFCOUNTER_IPACKETS:
                return (es->rx_unicast + es->rx_multicast + es->rx_broadcast);
        case IFCOUNTER_IERRORS:
                return (hs->crc_errors + hs->illegal_bytes +
                        hs->mac_local_faults + hs->mac_remote_faults +
                        hs->rx_undersize + hs->rx_oversize + hs->rx_fragments +
                        hs->rx_jabber);
        case IFCOUNTER_OPACKETS:
                return (es->tx_unicast + es->tx_multicast + es->tx_broadcast);
        case IFCOUNTER_OERRORS:
                return (if_get_counter_default(vsi->sc->ifp, counter) +
                    es->tx_errors);
        case IFCOUNTER_COLLISIONS:
                return (0);
        case IFCOUNTER_IBYTES:
                return (es->rx_bytes);
        case IFCOUNTER_OBYTES:
                return (es->tx_bytes);
        case IFCOUNTER_IMCASTS:
                return (es->rx_multicast);
        case IFCOUNTER_OMCASTS:
                return (es->tx_multicast);
        case IFCOUNTER_IQDROPS:
                return (es->rx_discards);
        case IFCOUNTER_OQDROPS:
                return (if_get_counter_default(vsi->sc->ifp, counter) +
                    hs->tx_dropped_link_down);
        case IFCOUNTER_NOPROTO:
                return (es->rx_unknown_protocol);
        default:
                return if_get_counter_default(vsi->sc->ifp, counter);
        }
}

/**
 * ice_save_pci_info - Save PCI configuration fields in HW struct
 * @hw: the ice_hw struct to save the PCI information in
 * @dev: the device to get the PCI information from
 *
 * This should only be called once, early in the device attach
 * process.
 */
void
ice_save_pci_info(struct ice_hw *hw, device_t dev)
{
        hw->vendor_id = pci_get_vendor(dev);
        hw->device_id = pci_get_device(dev);
        hw->subsystem_vendor_id = pci_get_subvendor(dev);
        hw->subsystem_device_id = pci_get_subdevice(dev);
        hw->revision_id = pci_get_revid(dev);
        hw->bus.device = pci_get_slot(dev);
        hw->bus.func = pci_get_function(dev);
}

/**
 * ice_replay_all_vsi_cfg - Replace configuration for all VSIs after reset
 * @sc: the device softc
 *
 * Replace the configuration for each VSI, and then cleanup replay
 * information. Called after a hardware reset in order to reconfigure the
 * active VSIs.
 */
int
ice_replay_all_vsi_cfg(struct ice_softc *sc)
{
        struct ice_hw *hw = &sc->hw;
        int status;
        int i;

        for (i = 0 ; i < sc->num_available_vsi; i++) {
                struct ice_vsi *vsi = sc->all_vsi[i];

                if (!vsi)
                        continue;

                status = ice_replay_vsi(hw, vsi->idx);
                if (status) {
                        device_printf(sc->dev, "Failed to replay VSI %d, err %s aq_err %s\n",
                                      vsi->idx, ice_status_str(status),
                                      ice_aq_str(hw->adminq.sq_last_status));
                        return (EIO);
                }
        }

        /* Cleanup replay filters after successful reconfiguration */
        ice_replay_post(hw);
        return (0);
}

/**
 * ice_clean_vsi_rss_cfg - Cleanup RSS configuration for a given VSI
 * @vsi: pointer to the VSI structure
 *
 * Cleanup the advanced RSS configuration for a given VSI. This is necessary
 * during driver removal to ensure that all RSS resources are properly
 * released.
 *
 * @remark this function doesn't report an error as it is expected to be
 * called during driver reset and unload, and there isn't much the driver can
 * do if freeing RSS resources fails.
 */
static void
ice_clean_vsi_rss_cfg(struct ice_vsi *vsi)
{
        struct ice_softc *sc = vsi->sc;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;

        status = ice_rem_vsi_rss_cfg(hw, vsi->idx);
        if (status)
                device_printf(dev,
                              "Failed to remove RSS configuration for VSI %d, err %s\n",
                              vsi->idx, ice_status_str(status));

        /* Remove this VSI from the RSS list */
        ice_rem_vsi_rss_list(hw, vsi->idx);
}

/**
 * ice_clean_all_vsi_rss_cfg - Cleanup RSS configuration for all VSIs
 * @sc: the device softc pointer
 *
 * Cleanup the advanced RSS configuration for all VSIs on a given PF
 * interface.
 *
 * @remark This should be called while preparing for a reset, to cleanup stale
 * RSS configuration for all VSIs.
 */
void
ice_clean_all_vsi_rss_cfg(struct ice_softc *sc)
{
        int i;

        /* No need to cleanup if RSS is not enabled */
        if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_RSS))
                return;

        for (i = 0; i < sc->num_available_vsi; i++) {
                struct ice_vsi *vsi = sc->all_vsi[i];

                if (vsi)
                        ice_clean_vsi_rss_cfg(vsi);
        }
}

/**
 * ice_requested_fec_mode - Return the requested FEC mode as a string
 * @pi: The port info structure
 *
 * Return a string representing the requested FEC mode.
 */
static const char *
ice_requested_fec_mode(struct ice_port_info *pi)
{
        struct ice_aqc_get_phy_caps_data pcaps = { 0 };
        int status;

        status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG,
                                     &pcaps, NULL);
        if (status)
                /* Just report unknown if we can't get capabilities */
                return "Unknown";

        /* Check if RS-FEC has been requested first */
        if (pcaps.link_fec_options & (ICE_AQC_PHY_FEC_25G_RS_528_REQ |
                                      ICE_AQC_PHY_FEC_25G_RS_544_REQ))
                return ice_fec_str(ICE_FEC_RS);

        /* If RS FEC has not been requested, then check BASE-R */
        if (pcaps.link_fec_options & (ICE_AQC_PHY_FEC_10G_KR_40G_KR4_REQ |
                                      ICE_AQC_PHY_FEC_25G_KR_REQ))
                return ice_fec_str(ICE_FEC_BASER);

        return ice_fec_str(ICE_FEC_NONE);
}

/**
 * ice_negotiated_fec_mode - Return the negotiated FEC mode as a string
 * @pi: The port info structure
 *
 * Return a string representing the current FEC mode.
 */
static const char *
ice_negotiated_fec_mode(struct ice_port_info *pi)
{
        /* First, check if RS has been requested first */
        if (pi->phy.link_info.fec_info & (ICE_AQ_LINK_25G_RS_528_FEC_EN |
                                          ICE_AQ_LINK_25G_RS_544_FEC_EN))
                return ice_fec_str(ICE_FEC_RS);

        /* If RS FEC has not been requested, then check BASE-R */
        if (pi->phy.link_info.fec_info & ICE_AQ_LINK_25G_KR_FEC_EN)
                return ice_fec_str(ICE_FEC_BASER);

        return ice_fec_str(ICE_FEC_NONE);
}

/**
 * ice_autoneg_mode - Return string indicating of autoneg completed
 * @pi: The port info structure
 *
 * Return "True" if autonegotiation is completed, "False" otherwise.
 */
static const char *
ice_autoneg_mode(struct ice_port_info *pi)
{
        if (pi->phy.link_info.an_info & ICE_AQ_AN_COMPLETED)
                return "True";
        else
                return "False";
}

/**
 * ice_flowcontrol_mode - Return string indicating the Flow Control mode
 * @pi: The port info structure
 *
 * Returns the current Flow Control mode as a string.
 */
static const char *
ice_flowcontrol_mode(struct ice_port_info *pi)
{
        return ice_fc_str(pi->fc.current_mode);
}

/**
 * ice_link_up_msg - Log a link up message with associated info
 * @sc: the device private softc
 *
 * Log a link up message with LOG_NOTICE message level. Include information
 * about the duplex, FEC mode, autonegotiation and flow control.
 */
void
ice_link_up_msg(struct ice_softc *sc)
{
        struct ice_hw *hw = &sc->hw;
        struct ifnet *ifp = sc->ifp;
        const char *speed, *req_fec, *neg_fec, *autoneg, *flowcontrol;

        speed = ice_aq_speed_to_str(hw->port_info);
        req_fec = ice_requested_fec_mode(hw->port_info);
        neg_fec = ice_negotiated_fec_mode(hw->port_info);
        autoneg = ice_autoneg_mode(hw->port_info);
        flowcontrol = ice_flowcontrol_mode(hw->port_info);

        log(LOG_NOTICE, "%s: Link is up, %s Full Duplex, Requested FEC: %s, Negotiated FEC: %s, Autoneg: %s, Flow Control: %s\n",
            if_name(ifp), speed, req_fec, neg_fec, autoneg, flowcontrol);
}

/**
 * ice_update_laa_mac - Update MAC address if Locally Administered
 * @sc: the device softc
 *
 * Update the device MAC address when a Locally Administered Address is
 * assigned.
 *
 * This function does *not* update the MAC filter list itself. Instead, it
 * should be called after ice_rm_pf_default_mac_filters, so that the previous
 * address filter will be removed, and before ice_cfg_pf_default_mac_filters,
 * so that the new address filter will be assigned.
 */
int
ice_update_laa_mac(struct ice_softc *sc)
{
        const u8 *lladdr = (const u8 *)if_getlladdr(sc->ifp);
        struct ice_hw *hw = &sc->hw;
        int status;

        /* If the address is the same, then there is nothing to update */
        if (!memcmp(lladdr, hw->port_info->mac.lan_addr, ETHER_ADDR_LEN))
                return (0);

        /* Reject Multicast addresses */
        if (ETHER_IS_MULTICAST(lladdr))
                return (EINVAL);

        status = ice_aq_manage_mac_write(hw, lladdr, ICE_AQC_MAN_MAC_UPDATE_LAA_WOL, NULL);
        if (status) {
                device_printf(sc->dev, "Failed to write mac %6D to firmware, err %s aq_err %s\n",
                              lladdr, ":", ice_status_str(status),
                              ice_aq_str(hw->adminq.sq_last_status));
                return (EFAULT);
        }

        /* Copy the address into place of the LAN address. */
        bcopy(lladdr, hw->port_info->mac.lan_addr, ETHER_ADDR_LEN);

        return (0);
}

/**
 * ice_get_and_print_bus_info - Save (PCI) bus info and print messages
 * @sc: device softc
 *
 * This will potentially print out a warning message if bus bandwidth
 * is insufficient for full-speed operation. This will not print out anything
 * for E82x devices since those are in SoCs, do not report valid PCIe info,
 * and cannot be moved to a different slot.
 *
 * This should only be called once, during the attach process, after
 * hw->port_info has been filled out with port link topology information
 * (from the Get PHY Capabilities Admin Queue command).
 */
void
ice_get_and_print_bus_info(struct ice_softc *sc)
{
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        u16 pci_link_status;
        int offset;

        if (!ice_is_e810(hw) && !ice_is_e830(hw))
                return;

        pci_find_cap(dev, PCIY_EXPRESS, &offset);
        pci_link_status = pci_read_config(dev, offset + PCIER_LINK_STA, 2);

        /* Fill out hw struct with PCIE link status info */
        ice_set_pci_link_status_data(hw, pci_link_status);

        /* Use info to print out bandwidth messages */
        ice_print_bus_link_data(dev, hw);

        if (ice_pcie_bandwidth_check(sc)) {
                device_printf(dev,
                    "PCI-Express bandwidth available for this device may be insufficient for optimal performance.\n");
                device_printf(dev,
                    "Please move the device to a different PCI-e link with more lanes and/or higher transfer rate.\n");
        }
}

/**
 * ice_pcie_bus_speed_to_rate - Convert driver bus speed enum value to
 * a 64-bit baudrate.
 * @speed: enum value to convert
 *
 * This only goes up to PCIE Gen 5.
 */
static uint64_t
ice_pcie_bus_speed_to_rate(enum ice_pcie_bus_speed speed)
{
        /* If the PCI-E speed is Gen1 or Gen2, then report
         * only 80% of bus speed to account for encoding overhead.
         */
        switch (speed) {
        case ice_pcie_speed_2_5GT:
                return IF_Gbps(2);
        case ice_pcie_speed_5_0GT:
                return IF_Gbps(4);
        case ice_pcie_speed_8_0GT:
                return IF_Gbps(8);
        case ice_pcie_speed_16_0GT:
                return IF_Gbps(16);
        case ice_pcie_speed_32_0GT:
                return IF_Gbps(32);
        case ice_pcie_speed_unknown:
        default:
                return 0;
        }
}

/**
 * ice_pcie_lnk_width_to_int - Convert driver pci-e width enum value to
 * a 32-bit number.
 * @width: enum value to convert
 */
static int
ice_pcie_lnk_width_to_int(enum ice_pcie_link_width width)
{
        switch (width) {
        case ice_pcie_lnk_x1:
                return (1);
        case ice_pcie_lnk_x2:
                return (2);
        case ice_pcie_lnk_x4:
                return (4);
        case ice_pcie_lnk_x8:
                return (8);
        case ice_pcie_lnk_x12:
                return (12);
        case ice_pcie_lnk_x16:
                return (16);
        case ice_pcie_lnk_x32:
                return (32);
        case ice_pcie_lnk_width_resrv:
        case ice_pcie_lnk_width_unknown:
        default:
                return (0);
        }
}

/**
 * ice_pcie_bandwidth_check - Check if PCI-E bandwidth is sufficient for
 * full-speed device operation.
 * @sc: adapter softc
 *
 * Returns 0 if sufficient; 1 if not.
 */
static uint8_t
ice_pcie_bandwidth_check(struct ice_softc *sc)
{
        struct ice_hw *hw = &sc->hw;
        int num_ports, pcie_width;
        u64 pcie_speed, port_speed;

        MPASS(hw->port_info);

        num_ports = bitcount32(hw->func_caps.common_cap.valid_functions);
        port_speed = ice_phy_types_to_max_rate(hw->port_info);
        pcie_speed = ice_pcie_bus_speed_to_rate(hw->bus.speed);
        pcie_width = ice_pcie_lnk_width_to_int(hw->bus.width);

        /*
         * If 2x100 on E810 or 2x200 on E830, clamp ports to 1 -- 2nd port is
         * intended for failover.
         */
        if ((port_speed >= IF_Gbps(100)) &&
            ((port_speed == IF_Gbps(100) && ice_is_e810(hw)) ||
             (port_speed == IF_Gbps(200) && ice_is_e830(hw))))
                num_ports = 1;

        return !!((num_ports * port_speed) > pcie_speed * pcie_width);
}

/**
 * ice_print_bus_link_data - Print PCI-E bandwidth information
 * @dev: device to print string for
 * @hw: hw struct with PCI-e link information
 */
static void
ice_print_bus_link_data(device_t dev, struct ice_hw *hw)
{
        device_printf(dev, "PCI Express Bus: Speed %s Width %s\n",
            ((hw->bus.speed == ice_pcie_speed_32_0GT) ? "32.0GT/s" :
            (hw->bus.speed == ice_pcie_speed_16_0GT) ? "16.0GT/s" :
            (hw->bus.speed == ice_pcie_speed_8_0GT) ? "8.0GT/s" :
            (hw->bus.speed == ice_pcie_speed_5_0GT) ? "5.0GT/s" :
            (hw->bus.speed == ice_pcie_speed_2_5GT) ? "2.5GT/s" : "Unknown"),
            (hw->bus.width == ice_pcie_lnk_x32) ? "x32" :
            (hw->bus.width == ice_pcie_lnk_x16) ? "x16" :
            (hw->bus.width == ice_pcie_lnk_x12) ? "x12" :
            (hw->bus.width == ice_pcie_lnk_x8) ? "x8" :
            (hw->bus.width == ice_pcie_lnk_x4) ? "x4" :
            (hw->bus.width == ice_pcie_lnk_x2) ? "x2" :
            (hw->bus.width == ice_pcie_lnk_x1) ? "x1" : "Unknown");
}

/**
 * ice_set_pci_link_status_data - store PCI bus info
 * @hw: pointer to hardware structure
 * @link_status: the link status word from PCI config space
 *
 * Stores the PCI bus info (speed, width, type) within the ice_hw structure
 **/
static void
ice_set_pci_link_status_data(struct ice_hw *hw, u16 link_status)
{
        u16 reg;

        hw->bus.type = ice_bus_pci_express;

        reg = (link_status & PCIEM_LINK_STA_WIDTH) >> 4;

        switch (reg) {
        case ice_pcie_lnk_x1:
        case ice_pcie_lnk_x2:
        case ice_pcie_lnk_x4:
        case ice_pcie_lnk_x8:
        case ice_pcie_lnk_x12:
        case ice_pcie_lnk_x16:
        case ice_pcie_lnk_x32:
                hw->bus.width = (enum ice_pcie_link_width)reg;
                break;
        default:
                hw->bus.width = ice_pcie_lnk_width_unknown;
                break;
        }

        reg = (link_status & PCIEM_LINK_STA_SPEED) + 0x13;

        switch (reg) {
        case ice_pcie_speed_2_5GT:
        case ice_pcie_speed_5_0GT:
        case ice_pcie_speed_8_0GT:
        case ice_pcie_speed_16_0GT:
        case ice_pcie_speed_32_0GT:
                hw->bus.speed = (enum ice_pcie_bus_speed)reg;
                break;
        default:
                hw->bus.speed = ice_pcie_speed_unknown;
                break;
        }
}

/**
 * ice_init_link_events - Initialize Link Status Events mask
 * @sc: the device softc
 *
 * Initialize the Link Status Events mask to disable notification of link
 * events we don't care about in software. Also request that link status
 * events be enabled.
 */
int
ice_init_link_events(struct ice_softc *sc)
{
        struct ice_hw *hw = &sc->hw;
        int status;
        u16 wanted_events;

        /* Set the bits for the events that we want to be notified by */
        wanted_events = (ICE_AQ_LINK_EVENT_UPDOWN |
                         ICE_AQ_LINK_EVENT_MEDIA_NA |
                         ICE_AQ_LINK_EVENT_MODULE_QUAL_FAIL);

        /* request that every event except the wanted events be masked */
        status = ice_aq_set_event_mask(hw, hw->port_info->lport, ~wanted_events, NULL);
        if (status) {
                device_printf(sc->dev,
                              "Failed to set link status event mask, err %s aq_err %s\n",
                              ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        /* Request link info with the LSE bit set to enable link status events */
        status = ice_aq_get_link_info(hw->port_info, true, NULL, NULL);
        if (status) {
                device_printf(sc->dev,
                              "Failed to enable link status events, err %s aq_err %s\n",
                              ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        return (0);
}

#ifndef GL_MDET_TX_TCLAN
/* Temporarily use this redefinition until the definition is fixed */
#define GL_MDET_TX_TCLAN        E800_GL_MDET_TX_TCLAN
#define PF_MDET_TX_TCLAN        E800_PF_MDET_TX_TCLAN
#endif /* !defined(GL_MDET_TX_TCLAN) */
/**
 * ice_handle_mdd_event - Handle possibly malicious events
 * @sc: the device softc
 *
 * Called by the admin task if an MDD detection interrupt is triggered.
 * Identifies possibly malicious events coming from VFs. Also triggers for
 * similar incorrect behavior from the PF as well.
 */
void
ice_handle_mdd_event(struct ice_softc *sc)
{
        struct ice_hw *hw = &sc->hw;
        bool mdd_detected = false, request_reinit = false;
        device_t dev = sc->dev;
        u32 reg;

        if (!ice_testandclear_state(&sc->state, ICE_STATE_MDD_PENDING))
                return;

        reg = rd32(hw, GL_MDET_TX_TCLAN);
        if (reg & GL_MDET_TX_TCLAN_VALID_M) {
                u8 pf_num  = (reg & GL_MDET_TX_TCLAN_PF_NUM_M) >> GL_MDET_TX_TCLAN_PF_NUM_S;
                u16 vf_num = (reg & GL_MDET_TX_TCLAN_VF_NUM_M) >> GL_MDET_TX_TCLAN_VF_NUM_S;
                u8 event   = (reg & GL_MDET_TX_TCLAN_MAL_TYPE_M) >> GL_MDET_TX_TCLAN_MAL_TYPE_S;
                u16 queue  = (reg & GL_MDET_TX_TCLAN_QNUM_M) >> GL_MDET_TX_TCLAN_QNUM_S;

                device_printf(dev, "Malicious Driver Detection Tx Descriptor check event '%s' on Tx queue %u PF# %u VF# %u\n",
                              ice_mdd_tx_tclan_str(event), queue, pf_num, vf_num);

                /* Only clear this event if it matches this PF, that way other
                 * PFs can read the event and determine VF and queue number.
                 */
                if (pf_num == hw->pf_id)
                        wr32(hw, GL_MDET_TX_TCLAN, 0xffffffff);

                mdd_detected = true;
        }

        /* Determine what triggered the MDD event */
        reg = rd32(hw, GL_MDET_TX_PQM);
        if (reg & GL_MDET_TX_PQM_VALID_M) {
                u8 pf_num  = (reg & GL_MDET_TX_PQM_PF_NUM_M) >> GL_MDET_TX_PQM_PF_NUM_S;
                u16 vf_num = (reg & GL_MDET_TX_PQM_VF_NUM_M) >> GL_MDET_TX_PQM_VF_NUM_S;
                u8 event   = (reg & GL_MDET_TX_PQM_MAL_TYPE_M) >> GL_MDET_TX_PQM_MAL_TYPE_S;
                u16 queue  = (reg & GL_MDET_TX_PQM_QNUM_M) >> GL_MDET_TX_PQM_QNUM_S;

                device_printf(dev, "Malicious Driver Detection Tx Quanta check event '%s' on Tx queue %u PF# %u VF# %u\n",
                              ice_mdd_tx_pqm_str(event), queue, pf_num, vf_num);

                /* Only clear this event if it matches this PF, that way other
                 * PFs can read the event and determine VF and queue number.
                 */
                if (pf_num == hw->pf_id)
                        wr32(hw, GL_MDET_TX_PQM, 0xffffffff);

                mdd_detected = true;
        }

        reg = rd32(hw, GL_MDET_RX);
        if (reg & GL_MDET_RX_VALID_M) {
                u8 pf_num  = (reg & GL_MDET_RX_PF_NUM_M) >> GL_MDET_RX_PF_NUM_S;
                u16 vf_num = (reg & GL_MDET_RX_VF_NUM_M) >> GL_MDET_RX_VF_NUM_S;
                u8 event   = (reg & GL_MDET_RX_MAL_TYPE_M) >> GL_MDET_RX_MAL_TYPE_S;
                u16 queue  = (reg & GL_MDET_RX_QNUM_M) >> GL_MDET_RX_QNUM_S;

                device_printf(dev, "Malicious Driver Detection Rx event '%s' on Rx queue %u PF# %u VF# %u\n",
                              ice_mdd_rx_str(event), queue, pf_num, vf_num);

                /* Only clear this event if it matches this PF, that way other
                 * PFs can read the event and determine VF and queue number.
                 */
                if (pf_num == hw->pf_id)
                        wr32(hw, GL_MDET_RX, 0xffffffff);

                mdd_detected = true;
        }

        /* Now, confirm that this event actually affects this PF, by checking
         * the PF registers.
         */
        if (mdd_detected) {
                reg = rd32(hw, PF_MDET_TX_TCLAN);
                if (reg & PF_MDET_TX_TCLAN_VALID_M) {
                        wr32(hw, PF_MDET_TX_TCLAN, 0xffff);
                        sc->soft_stats.tx_mdd_count++;
                        request_reinit = true;
                }

                reg = rd32(hw, PF_MDET_TX_PQM);
                if (reg & PF_MDET_TX_PQM_VALID_M) {
                        wr32(hw, PF_MDET_TX_PQM, 0xffff);
                        sc->soft_stats.tx_mdd_count++;
                        request_reinit = true;
                }

                reg = rd32(hw, PF_MDET_RX);
                if (reg & PF_MDET_RX_VALID_M) {
                        wr32(hw, PF_MDET_RX, 0xffff);
                        sc->soft_stats.rx_mdd_count++;
                        request_reinit = true;
                }
        }

        /* TODO: Implement logic to detect and handle events caused by VFs. */

        /* request that the upper stack re-initialize the Tx/Rx queues */
        if (request_reinit)
                ice_request_stack_reinit(sc);

        ice_flush(hw);
}

/**
 * ice_start_dcbx_agent - Start DCBX agent in FW via AQ command
 * @sc: the device softc
 *
 * @pre device is DCB capable and the FW LLDP agent has started
 *
 * Checks DCBX status and starts the DCBX agent if it is not in
 * a valid state via an AQ command.
 */
static void
ice_start_dcbx_agent(struct ice_softc *sc)
{
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        bool dcbx_agent_status;
        int status;

        hw->port_info->qos_cfg.dcbx_status = ice_get_dcbx_status(hw);

        if (hw->port_info->qos_cfg.dcbx_status != ICE_DCBX_STATUS_DONE &&
            hw->port_info->qos_cfg.dcbx_status != ICE_DCBX_STATUS_IN_PROGRESS) {
                /*
                 * Start DCBX agent, but not LLDP. The return value isn't
                 * checked here because a more detailed dcbx agent status is
                 * retrieved and checked in ice_init_dcb() and elsewhere.
                 */
                status = ice_aq_start_stop_dcbx(hw, true, &dcbx_agent_status, NULL);
                if (status && hw->adminq.sq_last_status != ICE_AQ_RC_EPERM)
                        device_printf(dev,
                            "start_stop_dcbx failed, err %s aq_err %s\n",
                            ice_status_str(status),
                            ice_aq_str(hw->adminq.sq_last_status));
        }
}

/**
 * ice_init_dcb_setup - Initialize DCB settings for HW
 * @sc: the device softc
 *
 * This needs to be called after the fw_lldp_agent sysctl is added, since that
 * can update the device's LLDP agent status if a tunable value is set.
 *
 * Get and store the initial state of DCB settings on driver load. Print out
 * informational messages as well.
 */
void
ice_init_dcb_setup(struct ice_softc *sc)
{
        struct ice_dcbx_cfg *local_dcbx_cfg;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;
        u8 pfcmode_ret;

        /* Don't do anything if DCB isn't supported */
        if (!ice_is_bit_set(sc->feat_cap, ICE_FEATURE_DCB)) {
                device_printf(dev, "%s: No DCB support\n", __func__);
                return;
        }

        /* Starts DCBX agent if it needs starting */
        ice_start_dcbx_agent(sc);

        /* This sets hw->port_info->qos_cfg.is_sw_lldp */
        status = ice_init_dcb(hw, true);

        /* If there is an error, then FW LLDP is not in a usable state */
        if (status != 0 && status != ICE_ERR_NOT_READY) {
                /* Don't print an error message if the return code from the AQ
                 * cmd performed in ice_init_dcb() is EPERM; that means the
                 * FW LLDP engine is disabled, and that is a valid state.
                 */
                if (!(status == ICE_ERR_AQ_ERROR &&
                      hw->adminq.sq_last_status == ICE_AQ_RC_EPERM)) {
                        device_printf(dev, "DCB init failed, err %s aq_err %s\n",
                                      ice_status_str(status),
                                      ice_aq_str(hw->adminq.sq_last_status));
                }
                hw->port_info->qos_cfg.dcbx_status = ICE_DCBX_STATUS_NOT_STARTED;
        }

        switch (hw->port_info->qos_cfg.dcbx_status) {
        case ICE_DCBX_STATUS_DIS:
                ice_debug(hw, ICE_DBG_DCB, "DCBX disabled\n");
                break;
        case ICE_DCBX_STATUS_NOT_STARTED:
                ice_debug(hw, ICE_DBG_DCB, "DCBX not started\n");
                break;
        case ICE_DCBX_STATUS_MULTIPLE_PEERS:
                ice_debug(hw, ICE_DBG_DCB, "DCBX detected multiple peers\n");
                break;
        default:
                break;
        }

        /* LLDP disabled in FW */
        if (hw->port_info->qos_cfg.is_sw_lldp) {
                ice_add_rx_lldp_filter(sc);
                device_printf(dev, "Firmware LLDP agent disabled\n");
        }

        /* Query and cache PFC mode */
        status = ice_aq_query_pfc_mode(hw, &pfcmode_ret, NULL);
        if (status) {
                device_printf(dev, "PFC mode query failed, err %s aq_err %s\n",
                              ice_status_str(status),
                              ice_aq_str(hw->adminq.sq_last_status));
        }
        local_dcbx_cfg = &hw->port_info->qos_cfg.local_dcbx_cfg;
        switch (pfcmode_ret) {
        case ICE_AQC_PFC_VLAN_BASED_PFC:
                local_dcbx_cfg->pfc_mode = ICE_QOS_MODE_VLAN;
                break;
        case ICE_AQC_PFC_DSCP_BASED_PFC:
                local_dcbx_cfg->pfc_mode = ICE_QOS_MODE_DSCP;
                break;
        default:
                /* DCB is disabled, but we shouldn't get here */
                break;
        }

        /* Set default SW MIB for init */
        ice_set_default_local_mib_settings(sc);

        ice_set_bit(ICE_FEATURE_DCB, sc->feat_en);
}

/**
 * ice_dcb_get_tc_map - Scans config to get bitmap of enabled TCs
 * @dcbcfg: DCB configuration to examine
 *
 * Scans a TC mapping table inside dcbcfg to find traffic classes
 * enabled and @returns a bitmask of enabled TCs
 */
u8
ice_dcb_get_tc_map(const struct ice_dcbx_cfg *dcbcfg)
{
        u8 tc_map = 0;
        int i = 0;

        switch (dcbcfg->pfc_mode) {
        case ICE_QOS_MODE_VLAN:
                /* XXX: "i" is actually "User Priority" here, not
                 * Traffic Class, but the max for both is 8, so it works
                 * out here.
                 */
                for (i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++)
                        tc_map |= BIT(dcbcfg->etscfg.prio_table[i]);
                break;
        case ICE_QOS_MODE_DSCP:
                for (i = 0; i < ICE_DSCP_NUM_VAL; i++)
                        tc_map |= BIT(dcbcfg->dscp_map[i]);
                break;
        default:
                /* Invalid Mode */
                tc_map = ICE_DFLT_TRAFFIC_CLASS;
                break;
        }

        return (tc_map);
}

/**
 * ice_dcb_get_num_tc - Get the number of TCs from DCBX config
 * @dcbcfg: config to retrieve number of TCs from
 *
 * @return number of contiguous TCs found in dcbcfg's ETS Configuration
 * Priority Assignment Table, a value from 1 to 8. If there are
 * non-contiguous TCs used (e.g. assigning 1 and 3 without using 2),
 * then returns 0.
 */
static u8
ice_dcb_get_num_tc(struct ice_dcbx_cfg *dcbcfg)
{
        u8 tc_map;

        tc_map = ice_dcb_get_tc_map(dcbcfg);

        return (ice_dcb_tc_contig(tc_map));
}

/**
 * ice_debug_print_mib_change_event - helper function to log LLDP MIB change events
 * @sc: the device private softc
 * @event: event received on a control queue
 *
 * Prints out the type and contents of an LLDP MIB change event in a DCB debug message.
 */
static void
ice_debug_print_mib_change_event(struct ice_softc *sc, struct ice_rq_event_info *event)
{
        struct ice_aqc_lldp_get_mib *params =
            (struct ice_aqc_lldp_get_mib *)&event->desc.params.lldp_get_mib;
        u8 mib_type, bridge_type, tx_status;

        static const char* mib_type_strings[] = {
            "Local MIB",
            "Remote MIB",
            "Reserved",
            "Reserved"
        };
        static const char* bridge_type_strings[] = {
            "Nearest Bridge",
            "Non-TPMR Bridge",
            "Reserved",
            "Reserved"
        };
        static const char* tx_status_strings[] = {
            "Port's TX active",
            "Port's TX suspended and drained",
            "Reserved",
            "Port's TX suspended and drained; blocked TC pipe flushed"
        };

        mib_type = (params->type & ICE_AQ_LLDP_MIB_TYPE_M) >>
            ICE_AQ_LLDP_MIB_TYPE_S;
        bridge_type = (params->type & ICE_AQ_LLDP_BRID_TYPE_M) >>
            ICE_AQ_LLDP_BRID_TYPE_S;
        tx_status = (params->type & ICE_AQ_LLDP_TX_M) >>
            ICE_AQ_LLDP_TX_S;

        ice_debug(&sc->hw, ICE_DBG_DCB, "LLDP MIB Change Event (%s, %s, %s)\n",
            mib_type_strings[mib_type], bridge_type_strings[bridge_type],
            tx_status_strings[tx_status]);

        /* Nothing else to report */
        if (!event->msg_buf)
                return;

        ice_debug(&sc->hw, ICE_DBG_DCB, "- %s contents:\n", mib_type_strings[mib_type]);
        ice_debug_array(&sc->hw, ICE_DBG_DCB, 16, 1, event->msg_buf,
                        event->msg_len);
}

/**
 * ice_dcb_needs_reconfig - Returns true if driver needs to reconfigure
 * @sc: the device private softc
 * @old_cfg: Old DCBX configuration to compare against
 * @new_cfg: New DCBX configuration to check
 *
 * @return true if something changed in new_cfg that requires the driver
 * to do some reconfiguration.
 */
static bool
ice_dcb_needs_reconfig(struct ice_softc *sc, struct ice_dcbx_cfg *old_cfg,
    struct ice_dcbx_cfg *new_cfg)
{
        struct ice_hw *hw = &sc->hw;
        bool needs_reconfig = false;

        /* No change detected in DCBX config */
        if (!memcmp(old_cfg, new_cfg, sizeof(*old_cfg))) {
                ice_debug(hw, ICE_DBG_DCB,
                    "No change detected in local DCBX configuration\n");
                return (false);
        }

        /* Check if ETS config has changed */
        if (memcmp(&new_cfg->etscfg, &old_cfg->etscfg,
                   sizeof(new_cfg->etscfg))) {
                /* If Priority Table has changed, then driver reconfig is needed */
                if (memcmp(&new_cfg->etscfg.prio_table,
                           &old_cfg->etscfg.prio_table,
                           sizeof(new_cfg->etscfg.prio_table))) {
                        ice_debug(hw, ICE_DBG_DCB, "ETS UP2TC changed\n");
                        needs_reconfig = true;
                }

                /* These are just informational */
                if (memcmp(&new_cfg->etscfg.tcbwtable,
                           &old_cfg->etscfg.tcbwtable,
                           sizeof(new_cfg->etscfg.tcbwtable))) {
                        ice_debug(hw, ICE_DBG_DCB, "ETS TCBW table changed\n");
                        needs_reconfig = true;
                }

                if (memcmp(&new_cfg->etscfg.tsatable,
                           &old_cfg->etscfg.tsatable,
                           sizeof(new_cfg->etscfg.tsatable))) {
                        ice_debug(hw, ICE_DBG_DCB, "ETS TSA table changed\n");
                        needs_reconfig = true;
                }
        }

        /* Check if PFC config has changed */
        if (memcmp(&new_cfg->pfc, &old_cfg->pfc, sizeof(new_cfg->pfc))) {
                ice_debug(hw, ICE_DBG_DCB, "PFC config changed\n");
                needs_reconfig = true;
        }

        /* Check if APP table has changed */
        if (memcmp(&new_cfg->app, &old_cfg->app, sizeof(new_cfg->app)))
                ice_debug(hw, ICE_DBG_DCB, "APP Table changed\n");

        ice_debug(hw, ICE_DBG_DCB, "%s result: %d\n", __func__, needs_reconfig);

        return (needs_reconfig);
}

/**
 * ice_stop_pf_vsi - Stop queues for PF LAN VSI
 * @sc: the device private softc
 *
 * Flushes interrupts and stops the queues associated with the PF LAN VSI.
 */
static void
ice_stop_pf_vsi(struct ice_softc *sc)
{
        /* Dissociate the Tx and Rx queues from the interrupts */
        ice_flush_txq_interrupts(&sc->pf_vsi);
        ice_flush_rxq_interrupts(&sc->pf_vsi);

        if (!ice_testandclear_state(&sc->state, ICE_STATE_DRIVER_INITIALIZED))
                return;

        /* Disable the Tx and Rx queues */
        ice_vsi_disable_tx(&sc->pf_vsi);
        ice_control_all_rx_queues(&sc->pf_vsi, false);
}

/**
 * ice_vsi_setup_q_map - Setup a VSI queue map
 * @vsi: the VSI being configured
 * @ctxt: VSI context structure
 */
static void
ice_vsi_setup_q_map(struct ice_vsi *vsi, struct ice_vsi_ctx *ctxt)
{
        u16 qcounts[ICE_MAX_TRAFFIC_CLASS] = {};
        u16 offset = 0, qmap = 0, pow = 0;
        u16 num_q_per_tc, qcount_rx, rem_queues;
        int i, j, k;

        if (vsi->num_tcs == 0) {
                /* at least TC0 should be enabled by default */
                vsi->num_tcs = 1;
                vsi->tc_map = 0x1;
        }

        qcount_rx = vsi->num_rx_queues;
        num_q_per_tc = min(qcount_rx / vsi->num_tcs, ICE_MAX_RXQS_PER_TC);

        if (!num_q_per_tc)
                num_q_per_tc = 1;

        /* Set initial values for # of queues to use for each active TC */
        ice_for_each_traffic_class(i)
                if (i < vsi->num_tcs)
                        qcounts[i] = num_q_per_tc;

        /* If any queues are unassigned, add them to TC 0 */
        rem_queues = qcount_rx % vsi->num_tcs;
        if (rem_queues > 0)
                qcounts[0] += rem_queues;

        /* TC mapping is a function of the number of Rx queues assigned to the
         * VSI for each traffic class and the offset of these queues.
         * The first 10 bits are for queue offset for TC0, next 4 bits for no:of
         * queues allocated to TC0. No:of queues is a power-of-2.
         *
         * If TC is not enabled, the queue offset is set to 0, and allocate one
         * queue, this way, traffic for the given TC will be sent to the default
         * queue.
         *
         * Setup number and offset of Rx queues for all TCs for the VSI
         */
        ice_for_each_traffic_class(i) {
                if (!(vsi->tc_map & BIT(i))) {
                        /* TC is not enabled */
                        vsi->tc_info[i].qoffset = 0;
                        vsi->tc_info[i].qcount_rx = 1;
                        vsi->tc_info[i].qcount_tx = 1;

                        ctxt->info.tc_mapping[i] = 0;
                        continue;
                }

                /* TC is enabled */
                vsi->tc_info[i].qoffset = offset;
                vsi->tc_info[i].qcount_rx = qcounts[i];
                vsi->tc_info[i].qcount_tx = qcounts[i];

                /* find the (rounded up) log-2 of queue count for current TC */
                pow = fls(qcounts[i] - 1);

                qmap = ((offset << ICE_AQ_VSI_TC_Q_OFFSET_S) &
                        ICE_AQ_VSI_TC_Q_OFFSET_M) |
                        ((pow << ICE_AQ_VSI_TC_Q_NUM_S) &
                         ICE_AQ_VSI_TC_Q_NUM_M);
                ctxt->info.tc_mapping[i] = CPU_TO_LE16(qmap);

                /* Store traffic class and handle data in queue structures */
                for (j = offset, k = 0; j < offset + qcounts[i]; j++, k++) {
                        vsi->tx_queues[j].q_handle = k;
                        vsi->tx_queues[j].tc = i;

                        vsi->rx_queues[j].tc = i;
                }
                
                offset += qcounts[i];
        }

        /* Rx queue mapping */
        ctxt->info.mapping_flags |= CPU_TO_LE16(ICE_AQ_VSI_Q_MAP_CONTIG);
        ctxt->info.q_mapping[0] = CPU_TO_LE16(vsi->rx_qmap[0]);
        ctxt->info.q_mapping[1] = CPU_TO_LE16(vsi->num_rx_queues);
}

/**
 * ice_pf_vsi_cfg_tc - Configure PF VSI for a given TC map
 * @sc: the device private softc
 * @tc_map: traffic class bitmap
 *
 * @pre VSI queues are stopped
 *
 * @return 0 if configuration is successful
 * @return EIO if Update VSI AQ cmd fails
 * @return ENODEV if updating Tx Scheduler fails
 */
static int
ice_pf_vsi_cfg_tc(struct ice_softc *sc, u8 tc_map)
{
        u16 max_txqs[ICE_MAX_TRAFFIC_CLASS] = { 0 };
        struct ice_vsi *vsi = &sc->pf_vsi;
        struct ice_hw *hw = &sc->hw;
        struct ice_vsi_ctx ctx = { 0 };
        device_t dev = sc->dev;
        int status;
        u8 num_tcs = 0;
        int i = 0;

        /* Count the number of enabled Traffic Classes */
        ice_for_each_traffic_class(i)
                if (tc_map & BIT(i))
                        num_tcs++;

        vsi->tc_map = tc_map;
        vsi->num_tcs = num_tcs;

        /* Set default parameters for context */
        ctx.vf_num = 0;
        ctx.info = vsi->info;

        /* Setup queue map */
        ice_vsi_setup_q_map(vsi, &ctx);

        /* Update VSI configuration in firmware (RX queues) */
        ctx.info.valid_sections = CPU_TO_LE16(ICE_AQ_VSI_PROP_RXQ_MAP_VALID);
        status = ice_update_vsi(hw, vsi->idx, &ctx, NULL);
        if (status) {
                device_printf(dev,
                    "%s: Update VSI AQ call failed, err %s aq_err %s\n",
                    __func__, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }
        vsi->info = ctx.info;

        /* Use values derived in ice_vsi_setup_q_map() */
        for (i = 0; i < num_tcs; i++)
                max_txqs[i] = vsi->tc_info[i].qcount_tx;

        if (hw->debug_mask & ICE_DBG_DCB) {
                device_printf(dev, "%s: max_txqs:", __func__);
                ice_for_each_traffic_class(i)
                        printf(" %d", max_txqs[i]);
                printf("\n");
        }

        /* Update LAN Tx queue info in firmware */
        status = ice_cfg_vsi_lan(hw->port_info, vsi->idx, vsi->tc_map,
                                 max_txqs);
        if (status) {
                device_printf(dev,
                    "%s: Failed VSI lan queue config, err %s aq_err %s\n",
                    __func__, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (ENODEV);
        }

        vsi->info.valid_sections = 0;

        return (0);
}

/**
 * ice_dcb_tc_contig - Count TCs if they're contiguous
 * @tc_map: pointer to priority table
 *
 * @return The number of traffic classes in
 * an 8-bit TC bitmap, or if there is a gap, then returns 0.
 */
static u8
ice_dcb_tc_contig(u8 tc_map)
{
        bool tc_unused = false;
        u8 ret = 0;

        /* Scan bitmask for contiguous TCs starting with TC0 */
        for (int i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++) {
                if (tc_map & BIT(i)) {
                        if (!tc_unused) {
                                ret++;
                        } else {
                                /* Non-contiguous TCs detected */
                                return (0);
                        }
                } else
                        tc_unused = true;
        }

        return (ret);
}

/**
 * ice_dcb_recfg - Reconfigure VSI with new DCB settings
 * @sc: the device private softc
 *
 * @pre All VSIs have been disabled/stopped
 * 
 * Reconfigures VSI settings based on local_dcbx_cfg.
 */
static void
ice_dcb_recfg(struct ice_softc *sc)
{
        struct ice_dcbx_cfg *dcbcfg =
            &sc->hw.port_info->qos_cfg.local_dcbx_cfg;
        device_t dev = sc->dev;
        u8 tc_map = 0;
        int ret;

        tc_map = ice_dcb_get_tc_map(dcbcfg);

        /* If non-contiguous TCs are used, then configure
         * the default TC instead. There's no support for
         * non-contiguous TCs being used.
         */
        if (ice_dcb_tc_contig(tc_map) == 0) {
                tc_map = ICE_DFLT_TRAFFIC_CLASS;
                ice_set_default_local_lldp_mib(sc);
        }

        /* Reconfigure VSI queues to add/remove traffic classes */
        ret = ice_pf_vsi_cfg_tc(sc, tc_map);
        if (ret)
                device_printf(dev,
                    "Failed to configure TCs for PF VSI, err %s\n",
                    ice_err_str(ret));

}

/**
 * ice_set_default_local_mib_settings - Set Local LLDP MIB to default settings
 * @sc: device softc structure
 *
 * Overwrites the driver's SW local LLDP MIB with default settings. This
 * ensures the driver has a valid MIB when it next uses the Set Local LLDP MIB
 * admin queue command.
 */
static void
ice_set_default_local_mib_settings(struct ice_softc *sc)
{
        struct ice_dcbx_cfg *dcbcfg;
        struct ice_hw *hw = &sc->hw;
        struct ice_port_info *pi;
        u8 maxtcs, maxtcs_ets, old_pfc_mode;

        pi = hw->port_info;

        dcbcfg = &pi->qos_cfg.local_dcbx_cfg;

        maxtcs = hw->func_caps.common_cap.maxtc;
        /* This value is only 3 bits; 8 TCs maps to 0 */
        maxtcs_ets = maxtcs & ICE_IEEE_ETS_MAXTC_M;

        /* VLAN vs DSCP mode needs to be preserved */
        old_pfc_mode = dcbcfg->pfc_mode;

        /**
         * Setup the default settings used by the driver for the Set Local
         * LLDP MIB Admin Queue command (0x0A08). (1TC w/ 100% BW, ETS, no
         * PFC, TSA=2).
         */
        memset(dcbcfg, 0, sizeof(*dcbcfg));

        dcbcfg->etscfg.willing = 1;
        dcbcfg->etscfg.tcbwtable[0] = 100;
        dcbcfg->etscfg.maxtcs = maxtcs_ets;
        dcbcfg->etscfg.tsatable[0] = 2;

        dcbcfg->etsrec = dcbcfg->etscfg;
        dcbcfg->etsrec.willing = 0;

        dcbcfg->pfc.willing = 1;
        dcbcfg->pfc.pfccap = maxtcs;

        dcbcfg->pfc_mode = old_pfc_mode;
}

/**
 * ice_do_dcb_reconfig - notify RDMA and reconfigure PF LAN VSI
 * @sc: the device private softc
 * @pending_mib: FW has a pending MIB change to execute
 * 
 * @pre Determined that the DCB configuration requires a change
 *
 * Reconfigures the PF LAN VSI based on updated DCB configuration
 * found in the hw struct's/port_info's/ local dcbx configuration.
 */
void
ice_do_dcb_reconfig(struct ice_softc *sc, bool pending_mib)
{
        struct ice_aqc_port_ets_elem port_ets = { 0 };
        struct ice_dcbx_cfg *local_dcbx_cfg;
        struct ice_hw *hw = &sc->hw;
        struct ice_port_info *pi;
        device_t dev = sc->dev;
        int status;

        pi = sc->hw.port_info;
        local_dcbx_cfg = &pi->qos_cfg.local_dcbx_cfg;

        ice_rdma_notify_dcb_qos_change(sc);
        /* If there's a pending MIB, tell the FW to execute the MIB change
         * now.
         */
        if (pending_mib) {
                status = ice_lldp_execute_pending_mib(hw);
                if ((status == ICE_ERR_AQ_ERROR) &&
                    (hw->adminq.sq_last_status == ICE_AQ_RC_ENOENT)) {
                        device_printf(dev,
                            "Execute Pending LLDP MIB AQ call failed, no pending MIB\n");
                } else if (status) {
                        device_printf(dev,
                            "Execute Pending LLDP MIB AQ call failed, err %s aq_err %s\n",
                            ice_status_str(status),
                            ice_aq_str(hw->adminq.sq_last_status));
                        /* This won't break traffic, but QoS will not work as expected */
                }
        }

        /* Set state when there's more than one TC */
        if (ice_dcb_get_num_tc(local_dcbx_cfg) > 1) {
                device_printf(dev, "Multiple traffic classes enabled\n");
                ice_set_state(&sc->state, ICE_STATE_MULTIPLE_TCS);
        } else {
                device_printf(dev, "Multiple traffic classes disabled\n");
                ice_clear_state(&sc->state, ICE_STATE_MULTIPLE_TCS);
        }

        /* Disable PF VSI since it's going to be reconfigured */
        ice_stop_pf_vsi(sc);

        /* Query ETS configuration and update SW Tx scheduler info */
        status = ice_query_port_ets(pi, &port_ets, sizeof(port_ets), NULL);
        if (status) {
                device_printf(dev,
                    "Query Port ETS AQ call failed, err %s aq_err %s\n",
                    ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                /* This won't break traffic, but QoS will not work as expected */
        }

        /* Change PF VSI configuration */
        ice_dcb_recfg(sc);

        /* Send new configuration to RDMA client driver */
        ice_rdma_dcb_qos_update(sc, pi);

        ice_request_stack_reinit(sc);
}

/**
 * ice_handle_mib_change_event - helper function to handle LLDP MIB change events
 * @sc: the device private softc
 * @event: event received on a control queue
 *
 * Checks the updated MIB it receives and possibly reconfigures the PF LAN
 * VSI depending on what has changed. This will also print out some debug
 * information about the MIB event if ICE_DBG_DCB is enabled in the debug_mask.
 */
static void
ice_handle_mib_change_event(struct ice_softc *sc, struct ice_rq_event_info *event)
{
        struct ice_aqc_lldp_get_mib *params =
            (struct ice_aqc_lldp_get_mib *)&event->desc.params.lldp_get_mib;
        struct ice_dcbx_cfg tmp_dcbx_cfg, *local_dcbx_cfg;
        struct ice_port_info *pi;
        device_t dev = sc->dev;
        struct ice_hw *hw = &sc->hw;
        bool needs_reconfig, mib_is_pending;
        int status;
        u8 mib_type, bridge_type;

        ASSERT_CFG_LOCKED(sc);

        ice_debug_print_mib_change_event(sc, event);

        pi = sc->hw.port_info;

        mib_type = (params->type & ICE_AQ_LLDP_MIB_TYPE_M) >>
            ICE_AQ_LLDP_MIB_TYPE_S;
        bridge_type = (params->type & ICE_AQ_LLDP_BRID_TYPE_M) >>
            ICE_AQ_LLDP_BRID_TYPE_S;
        mib_is_pending = (params->state & ICE_AQ_LLDP_MIB_CHANGE_STATE_M) >>
            ICE_AQ_LLDP_MIB_CHANGE_STATE_S;

        /* Ignore if event is not for Nearest Bridge */
        if (bridge_type != ICE_AQ_LLDP_BRID_TYPE_NEAREST_BRID)
                return;

        /* Check MIB Type and return if event for Remote MIB update */
        if (mib_type == ICE_AQ_LLDP_MIB_REMOTE) {
                /* Update the cached remote MIB and return */
                status = ice_aq_get_dcb_cfg(pi->hw, ICE_AQ_LLDP_MIB_REMOTE,
                                         ICE_AQ_LLDP_BRID_TYPE_NEAREST_BRID,
                                         &pi->qos_cfg.remote_dcbx_cfg);
                if (status)
                        device_printf(dev,
                            "%s: Failed to get Remote DCB config; status %s, aq_err %s\n",
                            __func__, ice_status_str(status),
                            ice_aq_str(hw->adminq.sq_last_status));
                /* Not fatal if this fails */
                return;
        }

        /* Save line length by aliasing the local dcbx cfg */
        local_dcbx_cfg = &pi->qos_cfg.local_dcbx_cfg;
        /* Save off the old configuration and clear current config */
        tmp_dcbx_cfg = *local_dcbx_cfg;
        memset(local_dcbx_cfg, 0, sizeof(*local_dcbx_cfg));

        /* Update the current local_dcbx_cfg with new data */
        if (mib_is_pending) {
                ice_get_dcb_cfg_from_mib_change(pi, event);
        } else {
                /* Get updated DCBX data from firmware */
                status = ice_get_dcb_cfg(pi);
                if (status) {
                        device_printf(dev,
                            "%s: Failed to get Local DCB config; status %s, aq_err %s\n",
                            __func__, ice_status_str(status),
                            ice_aq_str(hw->adminq.sq_last_status));
                        return;
                }
        }

        /* Check to see if DCB needs reconfiguring */
        needs_reconfig = ice_dcb_needs_reconfig(sc, &tmp_dcbx_cfg,
            local_dcbx_cfg);

        if (!needs_reconfig && !mib_is_pending)
                return;

        /* Reconfigure -- this will also notify FW that configuration is done,
         * if the FW MIB change is only pending instead of executed.
         */
        ice_do_dcb_reconfig(sc, mib_is_pending);
}

/**
 * ice_send_version - Send driver version to firmware
 * @sc: the device private softc
 *
 * Send the driver version to the firmware. This must be called as early as
 * possible after ice_init_hw().
 */
int
ice_send_version(struct ice_softc *sc)
{
        struct ice_driver_ver driver_version = {0};
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;

        driver_version.major_ver = ice_major_version;
        driver_version.minor_ver = ice_minor_version;
        driver_version.build_ver = ice_patch_version;
        driver_version.subbuild_ver = ice_rc_version;

        strlcpy((char *)driver_version.driver_string, ice_driver_version,
                sizeof(driver_version.driver_string));

        status = ice_aq_send_driver_ver(hw, &driver_version, NULL);
        if (status) {
                device_printf(dev, "Unable to send driver version to firmware, err %s aq_err %s\n",
                              ice_status_str(status), ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        return (0);
}

/**
 * ice_handle_lan_overflow_event - helper function to log LAN overflow events
 * @sc: device softc
 * @event: event received on a control queue
 *
 * Prints out a message when a LAN overflow event is detected on a receive
 * queue.
 */
static void
ice_handle_lan_overflow_event(struct ice_softc *sc, struct ice_rq_event_info *event)
{
        struct ice_aqc_event_lan_overflow *params =
            (struct ice_aqc_event_lan_overflow *)&event->desc.params.lan_overflow;
        struct ice_hw *hw = &sc->hw;

        ice_debug(hw, ICE_DBG_DCB, "LAN overflow event detected, prtdcb_ruptq=0x%08x, qtx_ctl=0x%08x\n",
                  LE32_TO_CPU(params->prtdcb_ruptq),
                  LE32_TO_CPU(params->qtx_ctl));
}

/**
 * ice_add_ethertype_to_list - Add an Ethertype filter to a filter list
 * @vsi: the VSI to target packets to
 * @list: the list to add the filter to
 * @ethertype: the Ethertype to filter on
 * @direction: The direction of the filter (Tx or Rx)
 * @action: the action to take
 *
 * Add an Ethertype filter to a filter list. Used to forward a series of
 * filters to the firmware for configuring the switch.
 *
 * Returns 0 on success, and an error code on failure.
 */
static int
ice_add_ethertype_to_list(struct ice_vsi *vsi, struct ice_list_head *list,
                          u16 ethertype, u16 direction,
                          enum ice_sw_fwd_act_type action)
{
        struct ice_fltr_list_entry *entry;

        MPASS((direction == ICE_FLTR_TX) || (direction == ICE_FLTR_RX));

        entry = (__typeof(entry))malloc(sizeof(*entry), M_ICE, M_NOWAIT|M_ZERO);
        if (!entry)
                return (ENOMEM);

        entry->fltr_info.flag = direction;
        entry->fltr_info.src_id = ICE_SRC_ID_VSI;
        entry->fltr_info.lkup_type = ICE_SW_LKUP_ETHERTYPE;
        entry->fltr_info.fltr_act = action;
        entry->fltr_info.vsi_handle = vsi->idx;
        entry->fltr_info.l_data.ethertype_mac.ethertype = ethertype;

        LIST_ADD(&entry->list_entry, list);

        return 0;
}

#define ETHERTYPE_PAUSE_FRAMES 0x8808
#define ETHERTYPE_LLDP_FRAMES 0x88cc

/**
 * ice_cfg_pf_ethertype_filters - Configure switch to drop ethertypes
 * @sc: the device private softc
 *
 * Configure the switch to drop PAUSE frames and LLDP frames transmitted from
 * the host. This prevents malicious VFs from sending these frames and being
 * able to control or configure the network.
 */
int
ice_cfg_pf_ethertype_filters(struct ice_softc *sc)
{
        struct ice_list_head ethertype_list;
        struct ice_vsi *vsi = &sc->pf_vsi;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;
        int err = 0;

        INIT_LIST_HEAD(&ethertype_list);

        /*
         * Note that the switch filters will ignore the VSI index for the drop
         * action, so we only need to program drop filters once for the main
         * VSI.
         */

        /* Configure switch to drop all Tx pause frames coming from any VSI. */
        if (sc->enable_tx_fc_filter) {
                err = ice_add_ethertype_to_list(vsi, &ethertype_list,
                                                ETHERTYPE_PAUSE_FRAMES,
                                                ICE_FLTR_TX, ICE_DROP_PACKET);
                if (err)
                        goto free_ethertype_list;
        }

        /* Configure switch to drop LLDP frames coming from any VSI */
        if (sc->enable_tx_lldp_filter) {
                err = ice_add_ethertype_to_list(vsi, &ethertype_list,
                                                ETHERTYPE_LLDP_FRAMES,
                                                ICE_FLTR_TX, ICE_DROP_PACKET);
                if (err)
                        goto free_ethertype_list;
        }

        status = ice_add_eth_mac(hw, &ethertype_list);
        if (status) {
                device_printf(dev,
                              "Failed to add Tx Ethertype filters, err %s aq_err %s\n",
                              ice_status_str(status),
                              ice_aq_str(hw->adminq.sq_last_status));
                err = (EIO);
        }

free_ethertype_list:
        ice_free_fltr_list(&ethertype_list);
        return err;
}

/**
 * ice_add_rx_lldp_filter - add ethertype filter for Rx LLDP frames
 * @sc: the device private structure
 *
 * Add a switch ethertype filter which forwards the LLDP frames to the main PF
 * VSI. Called when the fw_lldp_agent is disabled, to allow the LLDP frames to
 * be forwarded to the stack.
 */
void
ice_add_rx_lldp_filter(struct ice_softc *sc)
{
        struct ice_list_head ethertype_list;
        struct ice_vsi *vsi = &sc->pf_vsi;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;
        int err;
        u16 vsi_num;

        /*
         * If FW is new enough, use a direct AQ command to perform the filter
         * addition.
         */
        if (ice_fw_supports_lldp_fltr_ctrl(hw)) {
                vsi_num = ice_get_hw_vsi_num(hw, vsi->idx);
                status = ice_lldp_fltr_add_remove(hw, vsi_num, true);
                if (status) {
                        device_printf(dev,
                            "Failed to add Rx LLDP filter, err %s aq_err %s\n",
                            ice_status_str(status),
                            ice_aq_str(hw->adminq.sq_last_status));
                } else
                        ice_set_state(&sc->state,
                            ICE_STATE_LLDP_RX_FLTR_FROM_DRIVER);
                return;
        }

        INIT_LIST_HEAD(&ethertype_list);

        /* Forward Rx LLDP frames to the stack */
        err = ice_add_ethertype_to_list(vsi, &ethertype_list,
                                        ETHERTYPE_LLDP_FRAMES,
                                        ICE_FLTR_RX, ICE_FWD_TO_VSI);
        if (err) {
                device_printf(dev,
                              "Failed to add Rx LLDP filter, err %s\n",
                              ice_err_str(err));
                goto free_ethertype_list;
        }

        status = ice_add_eth_mac(hw, &ethertype_list);
        if (status && status != ICE_ERR_ALREADY_EXISTS) {
                device_printf(dev,
                              "Failed to add Rx LLDP filter, err %s aq_err %s\n",
                              ice_status_str(status),
                              ice_aq_str(hw->adminq.sq_last_status));
        } else {
                /*
                 * If status == ICE_ERR_ALREADY_EXISTS, we won't treat an
                 * already existing filter as an error case.
                 */
                ice_set_state(&sc->state, ICE_STATE_LLDP_RX_FLTR_FROM_DRIVER);
        }

free_ethertype_list:
        ice_free_fltr_list(&ethertype_list);
}

/**
 * ice_del_rx_lldp_filter - Remove ethertype filter for Rx LLDP frames
 * @sc: the device private structure
 *
 * Remove the switch filter forwarding LLDP frames to the main PF VSI, called
 * when the firmware LLDP agent is enabled, to stop routing LLDP frames to the
 * stack.
 */
static void
ice_del_rx_lldp_filter(struct ice_softc *sc)
{
        struct ice_list_head ethertype_list;
        struct ice_vsi *vsi = &sc->pf_vsi;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;
        int err;
        u16 vsi_num;

        /*
         * Only in the scenario where the driver added the filter during
         * this session (while the driver was loaded) would we be able to
         * delete this filter.
         */
        if (!ice_test_state(&sc->state, ICE_STATE_LLDP_RX_FLTR_FROM_DRIVER))
                return;

        /*
         * If FW is new enough, use a direct AQ command to perform the filter
         * removal.
         */
        if (ice_fw_supports_lldp_fltr_ctrl(hw)) {
                vsi_num = ice_get_hw_vsi_num(hw, vsi->idx);
                status = ice_lldp_fltr_add_remove(hw, vsi_num, false);
                if (status) {
                        device_printf(dev,
                            "Failed to remove Rx LLDP filter, err %s aq_err %s\n",
                            ice_status_str(status),
                            ice_aq_str(hw->adminq.sq_last_status));
                }
                return;
        }

        INIT_LIST_HEAD(&ethertype_list);

        /* Remove filter forwarding Rx LLDP frames to the stack */
        err = ice_add_ethertype_to_list(vsi, &ethertype_list,
                                        ETHERTYPE_LLDP_FRAMES,
                                        ICE_FLTR_RX, ICE_FWD_TO_VSI);
        if (err) {
                device_printf(dev,
                              "Failed to remove Rx LLDP filter, err %s\n",
                              ice_err_str(err));
                goto free_ethertype_list;
        }

        status = ice_remove_eth_mac(hw, &ethertype_list);
        if (status == ICE_ERR_DOES_NOT_EXIST) {
                ; /* Don't complain if we try to remove a filter that doesn't exist */
        } else if (status) {
                device_printf(dev,
                              "Failed to remove Rx LLDP filter, err %s aq_err %s\n",
                              ice_status_str(status),
                              ice_aq_str(hw->adminq.sq_last_status));
        }

free_ethertype_list:
        ice_free_fltr_list(&ethertype_list);
}

/**
 * ice_init_link_configuration -- Setup link in different ways depending
 * on whether media is available or not.
 * @sc: device private structure
 *
 * Called at the end of the attach process to either set default link
 * parameters if there is media available, or force HW link down and
 * set a state bit if there is no media.
 */
void
ice_init_link_configuration(struct ice_softc *sc)
{
        struct ice_port_info *pi = sc->hw.port_info;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status, retry_count = 0;

retry:
        pi->phy.get_link_info = true;
        status = ice_get_link_status(pi, &sc->link_up);

        if (status) {
                if (hw->adminq.sq_last_status == ICE_AQ_RC_EAGAIN) {
                        retry_count++;
                        ice_debug(hw, ICE_DBG_LINK,
                            "%s: ice_get_link_status failed with EAGAIN, attempt %d\n",
                            __func__, retry_count);
                        if (retry_count < ICE_LINK_AQ_MAX_RETRIES) {
                                ice_msec_pause(ICE_LINK_RETRY_DELAY);
                                goto retry;
                        }
                } else {
                        device_printf(dev,
                            "%s: ice_get_link_status failed; status %s, aq_err %s\n",
                            __func__, ice_status_str(status),
                            ice_aq_str(hw->adminq.sq_last_status));
                }
                return;
        }

        if (pi->phy.link_info.link_info & ICE_AQ_MEDIA_AVAILABLE) {
                ice_clear_state(&sc->state, ICE_STATE_NO_MEDIA);
                /* Apply default link settings */
                if (!ice_test_state(&sc->state, ICE_STATE_LINK_ACTIVE_ON_DOWN)) {
                        ice_set_link(sc, false);
                        ice_set_state(&sc->state, ICE_STATE_LINK_STATUS_REPORTED);
                } else
                        ice_apply_saved_phy_cfg(sc, ICE_APPLY_LS_FEC_FC);
        } else {
                 /* Set link down, and poll for media available in timer. This prevents the
                  * driver from receiving spurious link-related events.
                  */
                ice_set_state(&sc->state, ICE_STATE_NO_MEDIA);
                status = ice_aq_set_link_restart_an(pi, false, NULL);
                if (status && hw->adminq.sq_last_status != ICE_AQ_RC_EMODE)
                        device_printf(dev,
                            "%s: ice_aq_set_link_restart_an: status %s, aq_err %s\n",
                            __func__, ice_status_str(status),
                            ice_aq_str(hw->adminq.sq_last_status));
        }
}

/**
 * ice_apply_saved_phy_req_to_cfg -- Write saved user PHY settings to cfg data
 * @sc: device private structure
 * @cfg: new PHY config data to be modified
 *
 * Applies user settings for advertised speeds to the PHY type fields in the
 * supplied PHY config struct. It uses the data from pcaps to check if the
 * saved settings are invalid and uses the pcaps data instead if they are
 * invalid.
 */
static int
ice_apply_saved_phy_req_to_cfg(struct ice_softc *sc,
                               struct ice_aqc_set_phy_cfg_data *cfg)
{
        struct ice_phy_data phy_data = { 0 };
        struct ice_port_info *pi = sc->hw.port_info;
        u64 phy_low = 0, phy_high = 0;
        u16 link_speeds;
        int ret;

        link_speeds = pi->phy.curr_user_speed_req;

        if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_LINK_MGMT_VER_2)) {
                memset(&phy_data, 0, sizeof(phy_data));
                phy_data.report_mode = ICE_AQC_REPORT_DFLT_CFG;
                phy_data.user_speeds_orig = link_speeds;
                ret = ice_intersect_phy_types_and_speeds(sc, &phy_data);
                if (ret != 0) {
                        /* Error message already printed within function */
                        return (ret);
                }
                phy_low = phy_data.phy_low_intr;
                phy_high = phy_data.phy_high_intr;

                if (link_speeds == 0 || phy_data.user_speeds_intr)
                        goto finalize_link_speed;
                if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_LENIENT_LINK_MODE)) {
                        memset(&phy_data, 0, sizeof(phy_data));
                        phy_data.report_mode = ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA;
                        phy_data.user_speeds_orig = link_speeds;
                        ret = ice_intersect_phy_types_and_speeds(sc, &phy_data);
                        if (ret != 0) {
                                /* Error message already printed within function */
                                return (ret);
                        }
                        phy_low = phy_data.phy_low_intr;
                        phy_high = phy_data.phy_high_intr;

                        if (!phy_data.user_speeds_intr) {
                                phy_low = phy_data.phy_low_orig;
                                phy_high = phy_data.phy_high_orig;
                        }
                        goto finalize_link_speed;
                }
                /* If we're here, then it means the benefits of Version 2
                 * link management aren't utilized.  We fall through to
                 * handling Strict Link Mode the same as Version 1 link
                 * management.
                 */
        }

        memset(&phy_data, 0, sizeof(phy_data));
        if ((link_speeds == 0) &&
            (sc->ldo_tlv.phy_type_low || sc->ldo_tlv.phy_type_high))
                phy_data.report_mode = ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA;
        else
                phy_data.report_mode = ICE_AQC_REPORT_TOPO_CAP_MEDIA;
        phy_data.user_speeds_orig = link_speeds;
        ret = ice_intersect_phy_types_and_speeds(sc, &phy_data);
        if (ret != 0) {
                /* Error message already printed within function */
                return (ret);
        }
        phy_low = phy_data.phy_low_intr;
        phy_high = phy_data.phy_high_intr;

        if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_LENIENT_LINK_MODE)) {
                if (phy_low == 0 && phy_high == 0) {
                        device_printf(sc->dev,
                            "The selected speed is not supported by the current media. Please select a link speed that is supported by the current media.\n");
                        return (EINVAL);
                }
        } else {
                if (link_speeds == 0) {
                        if (sc->ldo_tlv.phy_type_low & phy_low ||
                            sc->ldo_tlv.phy_type_high & phy_high) {
                                phy_low &= sc->ldo_tlv.phy_type_low;
                                phy_high &= sc->ldo_tlv.phy_type_high;
                        }
                } else if (phy_low == 0 && phy_high == 0) {
                        memset(&phy_data, 0, sizeof(phy_data));
                        phy_data.report_mode = ICE_AQC_REPORT_TOPO_CAP_NO_MEDIA;
                        phy_data.user_speeds_orig = link_speeds;
                        ret = ice_intersect_phy_types_and_speeds(sc, &phy_data);
                        if (ret != 0) {
                                /* Error message already printed within function */
                                return (ret);
                        }
                        phy_low = phy_data.phy_low_intr;
                        phy_high = phy_data.phy_high_intr;

                        if (!phy_data.user_speeds_intr) {
                                phy_low = phy_data.phy_low_orig;
                                phy_high = phy_data.phy_high_orig;
                        }
                }
        }

finalize_link_speed:

        /* Cache new user settings for speeds */
        pi->phy.curr_user_speed_req = phy_data.user_speeds_intr;
        cfg->phy_type_low = htole64(phy_low);
        cfg->phy_type_high = htole64(phy_high);

        return (ret);
}

/**
 * ice_apply_saved_fec_req_to_cfg -- Write saved user FEC mode to cfg data
 * @sc: device private structure
 * @cfg: new PHY config data to be modified
 *
 * Applies user setting for FEC mode to PHY config struct. It uses the data
 * from pcaps to check if the saved settings are invalid and uses the pcaps
 * data instead if they are invalid.
 */
static int
ice_apply_saved_fec_req_to_cfg(struct ice_softc *sc,
                               struct ice_aqc_set_phy_cfg_data *cfg)
{
        struct ice_port_info *pi = sc->hw.port_info;
        int status;

        cfg->caps &= ~ICE_AQC_PHY_EN_AUTO_FEC;
        status = ice_cfg_phy_fec(pi, cfg, pi->phy.curr_user_fec_req);
        if (status)
                return (EIO);

        return (0);
}

/**
 * ice_apply_saved_fc_req_to_cfg -- Write saved user flow control mode to cfg data
 * @pi: port info struct
 * @cfg: new PHY config data to be modified
 *
 * Applies user setting for flow control mode to PHY config struct. There are
 * no invalid flow control mode settings; if there are, then this function
 * treats them like "ICE_FC_NONE".
 */
static void
ice_apply_saved_fc_req_to_cfg(struct ice_port_info *pi,
                              struct ice_aqc_set_phy_cfg_data *cfg)
{
        cfg->caps &= ~(ICE_AQ_PHY_ENA_TX_PAUSE_ABILITY |
                       ICE_AQ_PHY_ENA_RX_PAUSE_ABILITY);

        switch (pi->phy.curr_user_fc_req) {
        case ICE_FC_FULL:
                cfg->caps |= ICE_AQ_PHY_ENA_TX_PAUSE_ABILITY |
                             ICE_AQ_PHY_ENA_RX_PAUSE_ABILITY;
                break;
        case ICE_FC_RX_PAUSE:
                cfg->caps |= ICE_AQ_PHY_ENA_RX_PAUSE_ABILITY;
                break;
        case ICE_FC_TX_PAUSE:
                cfg->caps |= ICE_AQ_PHY_ENA_TX_PAUSE_ABILITY;
                break;
        default:
                /* ICE_FC_NONE */
                break;
        }
}

/**
 * ice_apply_saved_phy_cfg -- Re-apply user PHY config settings
 * @sc: device private structure
 * @settings: which settings to apply
 *
 * Applies user settings for advertised speeds, FEC mode, and flow
 * control mode to a PHY config struct; it uses the data from pcaps
 * to check if the saved settings are invalid and uses the pcaps
 * data instead if they are invalid.
 *
 * For things like sysctls where only one setting needs to be
 * updated, the bitmap allows the caller to specify which setting
 * to update.
 */
int
ice_apply_saved_phy_cfg(struct ice_softc *sc, u8 settings)
{
        struct ice_aqc_set_phy_cfg_data cfg = { 0 };
        struct ice_port_info *pi = sc->hw.port_info;
        struct ice_aqc_get_phy_caps_data pcaps = { 0 };
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        u64 phy_low, phy_high;
        int status;
        enum ice_fec_mode dflt_fec_mode;
        u16 dflt_user_speed;

        if (!settings || settings > ICE_APPLY_LS_FEC_FC) {
                ice_debug(hw, ICE_DBG_LINK, "Settings out-of-bounds: %u\n",
                    settings);
        }

        status = ice_aq_get_phy_caps(pi, false, ICE_AQC_REPORT_ACTIVE_CFG,
                                     &pcaps, NULL);
        if (status) {
                device_printf(dev,
                    "%s: ice_aq_get_phy_caps (ACTIVE) failed; status %s, aq_err %s\n",
                    __func__, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        phy_low = le64toh(pcaps.phy_type_low);
        phy_high = le64toh(pcaps.phy_type_high);

        /* Save off initial config parameters */
        dflt_user_speed = ice_aq_phy_types_to_link_speeds(phy_low, phy_high);
        dflt_fec_mode = ice_caps_to_fec_mode(pcaps.caps, pcaps.link_fec_options);

        /* Setup new PHY config */
        ice_copy_phy_caps_to_cfg(pi, &pcaps, &cfg);

        /* On error, restore active configuration values */
        if ((settings & ICE_APPLY_LS) &&
            ice_apply_saved_phy_req_to_cfg(sc, &cfg)) {
                pi->phy.curr_user_speed_req = dflt_user_speed;
                cfg.phy_type_low = pcaps.phy_type_low;
                cfg.phy_type_high = pcaps.phy_type_high;
        }
        if ((settings & ICE_APPLY_FEC) &&
            ice_apply_saved_fec_req_to_cfg(sc, &cfg)) {
                pi->phy.curr_user_fec_req = dflt_fec_mode;
        }
        if (settings & ICE_APPLY_FC) {
                /* No real error indicators for this process,
                 * so we'll just have to assume it works. */
                ice_apply_saved_fc_req_to_cfg(pi, &cfg);
        }

        /* Enable link and re-negotiate it */
        cfg.caps |= ICE_AQ_PHY_ENA_AUTO_LINK_UPDT | ICE_AQ_PHY_ENA_LINK;

        status = ice_aq_set_phy_cfg(hw, pi, &cfg, NULL);
        if (status) {
                /* Don't indicate failure if there's no media in the port.
                 * The settings have been saved and will apply when media
                 * is inserted.
                 */
                if ((status == ICE_ERR_AQ_ERROR) &&
                    (hw->adminq.sq_last_status == ICE_AQ_RC_EBUSY)) {
                        device_printf(dev,
                            "%s: Setting will be applied when media is inserted\n",
                            __func__);
                        return (0);
                } else {
                        device_printf(dev,
                            "%s: ice_aq_set_phy_cfg failed; status %s, aq_err %s\n",
                            __func__, ice_status_str(status),
                            ice_aq_str(hw->adminq.sq_last_status));
                        return (EIO);
                }
        }

        return (0);
}

/**
 * ice_print_ldo_tlv - Print out LDO TLV information
 * @sc: device private structure
 * @tlv: LDO TLV information from the adapter NVM
 *
 * Dump out the information in tlv to the kernel message buffer; intended for
 * debugging purposes.
 */
static void
ice_print_ldo_tlv(struct ice_softc *sc, struct ice_link_default_override_tlv *tlv)
{
        device_t dev = sc->dev;

        device_printf(dev, "TLV: -options     0x%02x\n", tlv->options);
        device_printf(dev, "     -phy_config  0x%02x\n", tlv->phy_config);
        device_printf(dev, "     -fec_options 0x%02x\n", tlv->fec_options);
        device_printf(dev, "     -phy_high    0x%016llx\n",
            (unsigned long long)tlv->phy_type_high);
        device_printf(dev, "     -phy_low     0x%016llx\n",
            (unsigned long long)tlv->phy_type_low);
}

/**
 * ice_set_link_management_mode -- Strict or lenient link management
 * @sc: device private structure
 *
 * Some NVMs give the adapter the option to advertise a superset of link
 * configurations.  This checks to see if that option is enabled.
 * Further, the NVM could also provide a specific set of configurations
 * to try; these are cached in the driver's private structure if they
 * are available.
 */
void
ice_set_link_management_mode(struct ice_softc *sc)
{
        struct ice_port_info *pi = sc->hw.port_info;
        device_t dev = sc->dev;
        struct ice_link_default_override_tlv tlv = { 0 };
        int status;

        /* Port must be in strict mode if FW version is below a certain
         * version. (i.e. Don't set lenient mode features)
         */
        if (!(ice_fw_supports_link_override(&sc->hw)))
                return;

        status = ice_get_link_default_override(&tlv, pi);
        if (status) {
                device_printf(dev,
                    "%s: ice_get_link_default_override failed; status %s, aq_err %s\n",
                    __func__, ice_status_str(status),
                    ice_aq_str(sc->hw.adminq.sq_last_status));
                return;
        }

        if (sc->hw.debug_mask & ICE_DBG_LINK)
                ice_print_ldo_tlv(sc, &tlv);

        /* Set lenient link mode */
        if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_LENIENT_LINK_MODE) &&
            (!(tlv.options & ICE_LINK_OVERRIDE_STRICT_MODE)))
                ice_set_bit(ICE_FEATURE_LENIENT_LINK_MODE, sc->feat_en);

        /* FW supports reporting a default configuration */
        if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_LINK_MGMT_VER_2) &&
            ice_fw_supports_report_dflt_cfg(&sc->hw)) {
                ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_2, sc->feat_en);
                /* Knowing we're at a high enough firmware revision to
                 * support this link management configuration, we don't
                 * need to check/support earlier versions.
                 */
                return;
        }

        /* Default overrides only work if in lenient link mode */
        if (ice_is_bit_set(sc->feat_cap, ICE_FEATURE_LINK_MGMT_VER_1) &&
            ice_is_bit_set(sc->feat_en, ICE_FEATURE_LENIENT_LINK_MODE) &&
            (tlv.options & ICE_LINK_OVERRIDE_EN))
                ice_set_bit(ICE_FEATURE_LINK_MGMT_VER_1, sc->feat_en);

        /* Cache the LDO TLV structure in the driver, since it
         * won't change during the driver's lifetime.
         */
        sc->ldo_tlv = tlv;
}

/**
 * ice_set_link -- Set up/down link on phy
 * @sc: device private structure
 * @enabled: link status to set up
 *
 * This should be called when change of link status is needed.
 */
void
ice_set_link(struct ice_softc *sc, bool enabled)
{
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;

        if (ice_driver_is_detaching(sc))
                return;

        if (ice_test_state(&sc->state, ICE_STATE_NO_MEDIA))
                return;

        if (enabled)
                ice_apply_saved_phy_cfg(sc, ICE_APPLY_LS_FEC_FC);
        else {
                status = ice_aq_set_link_restart_an(hw->port_info, false, NULL);
                if (status) {
                        if (hw->adminq.sq_last_status == ICE_AQ_RC_EMODE)
                                device_printf(dev,
                                    "%s: Link control not enabled in current device mode\n",
                                    __func__);
                        else
                                device_printf(dev,
                                    "%s: ice_aq_set_link_restart_an: status %s, aq_err %s\n",
                                    __func__, ice_status_str(status),
                                    ice_aq_str(hw->adminq.sq_last_status));
                } else
                        sc->link_up = false;
        }
}

/**
 * ice_init_saved_phy_cfg -- Set cached user PHY cfg settings with NVM defaults
 * @sc: device private structure
 *
 * This should be called before the tunables for these link settings
 * (e.g. advertise_speed) are added -- so that these defaults don't overwrite
 * the cached values that the sysctl handlers will write.
 *
 * This also needs to be called before ice_init_link_configuration, to ensure
 * that there are sane values that can be written if there is media available
 * in the port.
 */
void
ice_init_saved_phy_cfg(struct ice_softc *sc)
{
        struct ice_port_info *pi = sc->hw.port_info;
        struct ice_aqc_get_phy_caps_data pcaps = { 0 };
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;
        u64 phy_low, phy_high;
        u8 report_mode = ICE_AQC_REPORT_TOPO_CAP_MEDIA;

        if (ice_is_bit_set(sc->feat_en, ICE_FEATURE_LINK_MGMT_VER_2))
                report_mode = ICE_AQC_REPORT_DFLT_CFG;
        status = ice_aq_get_phy_caps(pi, false, report_mode, &pcaps, NULL);
        if (status) {
                device_printf(dev,
                    "%s: ice_aq_get_phy_caps (%s) failed; status %s, aq_err %s\n",
                    __func__,
                    report_mode == ICE_AQC_REPORT_DFLT_CFG ? "DFLT" : "w/MEDIA",
                    ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return;
        }

        phy_low = le64toh(pcaps.phy_type_low);
        phy_high = le64toh(pcaps.phy_type_high);

        /* Save off initial config parameters */
        pi->phy.curr_user_speed_req =
           ice_aq_phy_types_to_link_speeds(phy_low, phy_high);
        pi->phy.curr_user_fec_req = ice_caps_to_fec_mode(pcaps.caps,
            pcaps.link_fec_options);
        pi->phy.curr_user_fc_req = ice_caps_to_fc_mode(pcaps.caps);
}

/**
 * ice_module_init - Driver callback to handle module load
 *
 * Callback for handling module load events. This function should initialize
 * any data structures that are used for the life of the device driver.
 */
static int
ice_module_init(void)
{
        ice_rdma_init();
        return (0);
}

/**
 * ice_module_exit - Driver callback to handle module exit
 *
 * Callback for handling module unload events. This function should release
 * any resources initialized during ice_module_init.
 *
 * If this function returns non-zero, the module will not be unloaded. It
 * should only return such a value if the module cannot be unloaded at all,
 * such as due to outstanding memory references that cannot be revoked.
 */
static int
ice_module_exit(void)
{
        ice_rdma_exit();
        return (0);
}

/**
 * ice_module_event_handler - Callback for module events
 * @mod: unused module_t parameter
 * @what: the event requested
 * @arg: unused event argument
 *
 * Callback used to handle module events from the stack. Used to allow the
 * driver to define custom behavior that should happen at module load and
 * unload.
 */
int
ice_module_event_handler(module_t __unused mod, int what, void __unused *arg)
{
        switch (what) {
        case MOD_LOAD:
                return ice_module_init();
        case MOD_UNLOAD:
                return ice_module_exit();
        default:
                /* TODO: do we need to handle MOD_QUIESCE and MOD_SHUTDOWN? */
                return (EOPNOTSUPP);
        }
}

/**
 * ice_handle_nvm_access_ioctl - Handle an NVM access ioctl request
 * @sc: the device private softc
 * @ifd: ifdrv ioctl request pointer
 */
int
ice_handle_nvm_access_ioctl(struct ice_softc *sc, struct ifdrv *ifd)
{
        union ice_nvm_access_data *data;
        struct ice_nvm_access_cmd *cmd;
        size_t ifd_len = ifd->ifd_len, malloc_len;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;
        u8 *nvm_buffer;
        int err;

        /*
         * ifioctl forwards SIOCxDRVSPEC to iflib without performing
         * a privilege check. In turn, iflib forwards the ioctl to the driver
         * without performing a privilege check. Perform one here to ensure
         * that non-privileged threads cannot access this interface.
         */
        err = priv_check(curthread, PRIV_DRIVER);
        if (err)
                return (err);

        if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) {
                device_printf(dev, "%s: Driver must rebuild data structures after a reset. Operation aborted.\n",
                              __func__);
                return (EBUSY);
        }

        if (ifd_len < sizeof(struct ice_nvm_access_cmd)) {
                device_printf(dev, "%s: ifdrv length is too small. Got %zu, but expected %zu\n",
                              __func__, ifd_len, sizeof(struct ice_nvm_access_cmd));
                return (EINVAL);
        }

        if (ifd->ifd_data == NULL) {
                device_printf(dev, "%s: ifd data buffer not present.\n",
                              __func__);
                return (EINVAL);
        }

        /*
         * If everything works correctly, ice_handle_nvm_access should not
         * modify data past the size of the ioctl length. However, it could
         * lead to memory corruption if it did. Make sure to allocate at least
         * enough space for the command and data regardless. This
         * ensures that any access to the data union will not access invalid
         * memory.
         */
        malloc_len = max(ifd_len, sizeof(*data) + sizeof(*cmd));

        nvm_buffer = (u8 *)malloc(malloc_len, M_ICE, M_ZERO | M_WAITOK);
        if (!nvm_buffer)
                return (ENOMEM);

        /* Copy the NVM access command and data in from user space */
        /* coverity[tainted_data_argument] */
        err = copyin(ifd->ifd_data, nvm_buffer, ifd_len);
        if (err) {
                device_printf(dev, "%s: Copying request from user space failed, err %s\n",
                              __func__, ice_err_str(err));
                goto cleanup_free_nvm_buffer;
        }

        /*
         * The NVM command structure is immediately followed by data which
         * varies in size based on the command.
         */
        cmd = (struct ice_nvm_access_cmd *)nvm_buffer;
        data = (union ice_nvm_access_data *)(nvm_buffer + sizeof(struct ice_nvm_access_cmd));

        /* Handle the NVM access request */
        status = ice_handle_nvm_access(hw, cmd, data);
        if (status)
                ice_debug(hw, ICE_DBG_NVM,
                          "NVM access request failed, err %s\n",
                          ice_status_str(status));

        /* Copy the possibly modified contents of the handled request out */
        err = copyout(nvm_buffer, ifd->ifd_data, ifd_len);
        if (err) {
                device_printf(dev, "%s: Copying response back to user space failed, err %s\n",
                              __func__, ice_err_str(err));
                goto cleanup_free_nvm_buffer;
        }

        /* Convert private status to an error code for proper ioctl response */
        switch (status) {
        case 0:
                err = (0);
                break;
        case ICE_ERR_NO_MEMORY:
                err = (ENOMEM);
                break;
        case ICE_ERR_OUT_OF_RANGE:
                err = (ENOTTY);
                break;
        case ICE_ERR_PARAM:
        default:
                err = (EINVAL);
                break;
        }

cleanup_free_nvm_buffer:
        free(nvm_buffer, M_ICE);
        return err;
}

/**
 * ice_read_sff_eeprom - Read data from SFF eeprom
 * @sc: device softc
 * @dev_addr: I2C device address (typically 0xA0 or 0xA2)
 * @offset: offset into the eeprom
 * @data: pointer to data buffer to store read data in
 * @length: length to read; max length is 16
 *
 * Read from the SFF eeprom in the module for this PF's port. For more details
 * on the contents of an SFF eeprom, refer to SFF-8724 (SFP), SFF-8636 (QSFP),
 * and SFF-8024 (both).
 */
int
ice_read_sff_eeprom(struct ice_softc *sc, u16 dev_addr, u16 offset, u8* data, u16 length)
{
        struct ice_hw *hw = &sc->hw;
        int ret = 0, retries = 0;
        int status;

        if (length > 16)
                return (EINVAL);

        if (ice_test_state(&sc->state, ICE_STATE_RECOVERY_MODE))
                return (ENOSYS);

        if (ice_test_state(&sc->state, ICE_STATE_NO_MEDIA))
                return (ENXIO);

        do {
                status = ice_aq_sff_eeprom(hw, 0, dev_addr,
                                           offset, 0, 0, data, length,
                                           false, NULL);
                if (!status) {
                        ret = 0;
                        break;
                }
                if (status == ICE_ERR_AQ_ERROR &&
                    hw->adminq.sq_last_status == ICE_AQ_RC_EBUSY) {
                        ret = EBUSY;
                        continue;
                }
                if (status == ICE_ERR_AQ_ERROR &&
                    hw->adminq.sq_last_status == ICE_AQ_RC_EACCES) {
                        /* FW says I2C access isn't supported */
                        ret = EACCES;
                        break;
                }
                if (status == ICE_ERR_AQ_ERROR &&
                    hw->adminq.sq_last_status == ICE_AQ_RC_EPERM) {
                        device_printf(sc->dev,
                                  "%s: Module pointer location specified in command does not permit the required operation.\n",
                                  __func__);
                        ret = EPERM;
                        break;
                } else {
                        device_printf(sc->dev,
                                  "%s: Error reading I2C data: err %s aq_err %s\n",
                                  __func__, ice_status_str(status),
                                  ice_aq_str(hw->adminq.sq_last_status));
                        ret = EIO;
                        break;
                }
        } while (retries++ < ICE_I2C_MAX_RETRIES);

        if (ret == EBUSY)
                device_printf(sc->dev,
                          "%s: Error reading I2C data after %d retries\n",
                          __func__, ICE_I2C_MAX_RETRIES);

        return (ret);
}

/**
 * ice_handle_i2c_req - Driver independent I2C request handler
 * @sc: device softc
 * @req: The I2C parameters to use
 *
 * Read from the port's I2C eeprom using the parameters from the ioctl.
 */
int
ice_handle_i2c_req(struct ice_softc *sc, struct ifi2creq *req)
{
        return ice_read_sff_eeprom(sc, req->dev_addr, req->offset, req->data, req->len);
}

/**
 * ice_sysctl_read_i2c_diag_data - Read some module diagnostic data via i2c
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Read 8 bytes of diagnostic data from the SFF eeprom in the (Q)SFP module
 * inserted into the port.
 *
 *             | SFP A2  | QSFP Lower Page
 * ------------|---------|----------------
 * Temperature | 96-97   | 22-23
 * Vcc         | 98-99   | 26-27
 * TX power    | 102-103 | 34-35..40-41
 * RX power    | 104-105 | 50-51..56-57
 */
static int
ice_sysctl_read_i2c_diag_data(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        device_t dev = sc->dev;
        struct sbuf *sbuf;
        int ret;
        u8 data[16];

        UNREFERENCED_PARAMETER(arg2);
        UNREFERENCED_PARAMETER(oidp);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        if (req->oldptr == NULL) {
                ret = SYSCTL_OUT(req, 0, 128);
                return (ret);
        }

        ret = ice_read_sff_eeprom(sc, 0xA0, 0, data, 1);
        if (ret)
                return (ret);

        /* 0x3 for SFP; 0xD/0x11 for QSFP+/QSFP28 */
        if (data[0] == 0x3) {
                /*
                 * Check for:
                 * - Internally calibrated data
                 * - Diagnostic monitoring is implemented
                 */
                ice_read_sff_eeprom(sc, 0xA0, 92, data, 1);
                if (!(data[0] & 0x60)) {
                        device_printf(dev, "Module doesn't support diagnostics: 0xA0[92] = %02X\n", data[0]);
                        return (ENODEV);
                }

                sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);

                ice_read_sff_eeprom(sc, 0xA2, 96, data, 4);
                for (int i = 0; i < 4; i++)
                        sbuf_printf(sbuf, "%02X ", data[i]);

                ice_read_sff_eeprom(sc, 0xA2, 102, data, 4);
                for (int i = 0; i < 4; i++)
                        sbuf_printf(sbuf, "%02X ", data[i]);
        } else if (data[0] == 0xD || data[0] == 0x11) {
                /*
                 * QSFP+ modules are always internally calibrated, and must indicate
                 * what types of diagnostic monitoring are implemented
                 */
                sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);

                ice_read_sff_eeprom(sc, 0xA0, 22, data, 2);
                for (int i = 0; i < 2; i++)
                        sbuf_printf(sbuf, "%02X ", data[i]);

                ice_read_sff_eeprom(sc, 0xA0, 26, data, 2);
                for (int i = 0; i < 2; i++)
                        sbuf_printf(sbuf, "%02X ", data[i]);

                ice_read_sff_eeprom(sc, 0xA0, 34, data, 2);
                for (int i = 0; i < 2; i++)
                        sbuf_printf(sbuf, "%02X ", data[i]);

                ice_read_sff_eeprom(sc, 0xA0, 50, data, 2);
                for (int i = 0; i < 2; i++)
                        sbuf_printf(sbuf, "%02X ", data[i]);
        } else {
                device_printf(dev, "Module is not SFP/SFP+/SFP28/QSFP+ (%02X)\n", data[0]);
                return (ENODEV);
        }

        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        return (0);
}

/**
 * ice_alloc_intr_tracking - Setup interrupt tracking structures
 * @sc: device softc structure
 *
 * Sets up the resource manager for keeping track of interrupt allocations,
 * and initializes the tracking maps for the PF's interrupt allocations.
 *
 * Unlike the scheme for queues, this is done in one step since both the
 * manager and the maps both have the same lifetime.
 *
 * @returns 0 on success, or an error code on failure.
 */
int
ice_alloc_intr_tracking(struct ice_softc *sc)
{
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int err;

        if (hw->func_caps.common_cap.num_msix_vectors > ICE_MAX_MSIX_VECTORS) {
                device_printf(dev, "%s: Invalid num_msix_vectors value (%u) received from FW.\n",
                        __func__,
                        hw->func_caps.common_cap.num_msix_vectors);
                return (EINVAL);
        }

        /* Initialize the interrupt allocation manager */
        err = ice_resmgr_init_contig_only(&sc->dev_imgr,
            hw->func_caps.common_cap.num_msix_vectors);
        if (err) {
                device_printf(dev, "Unable to initialize PF interrupt manager: %s\n",
                              ice_err_str(err));
                return (err);
        }

        /* Allocate PF interrupt mapping storage */
        if (!(sc->pf_imap =
              (u16 *)malloc(sizeof(u16) * hw->func_caps.common_cap.num_msix_vectors,
              M_ICE, M_NOWAIT))) {
                device_printf(dev, "Unable to allocate PF imap memory\n");
                err = ENOMEM;
                goto free_imgr;
        }
        if (!(sc->rdma_imap =
              (u16 *)malloc(sizeof(u16) * hw->func_caps.common_cap.num_msix_vectors,
              M_ICE, M_NOWAIT))) {
                device_printf(dev, "Unable to allocate RDMA imap memory\n");
                err = ENOMEM;
                free(sc->pf_imap, M_ICE);
                goto free_imgr;
        }
        for (u32 i = 0; i < hw->func_caps.common_cap.num_msix_vectors; i++) {
                sc->pf_imap[i] = ICE_INVALID_RES_IDX;
                sc->rdma_imap[i] = ICE_INVALID_RES_IDX;
        }

        return (0);

free_imgr:
        ice_resmgr_destroy(&sc->dev_imgr);
        return (err);
}

/**
 * ice_free_intr_tracking - Free PF interrupt tracking structures
 * @sc: device softc structure
 *
 * Frees the interrupt resource allocation manager and the PF's owned maps.
 *
 * VF maps are released when the owning VF's are destroyed, which should always
 * happen before this function is called.
 */
void
ice_free_intr_tracking(struct ice_softc *sc)
{
        if (sc->pf_imap) {
                ice_resmgr_release_map(&sc->dev_imgr, sc->pf_imap,
                                       sc->lan_vectors);
                free(sc->pf_imap, M_ICE);
                sc->pf_imap = NULL;
        }
        if (sc->rdma_imap) {
                ice_resmgr_release_map(&sc->dev_imgr, sc->rdma_imap,
                                       sc->lan_vectors);
                free(sc->rdma_imap, M_ICE);
                sc->rdma_imap = NULL;
        }

        ice_resmgr_destroy(&sc->dev_imgr);

        ice_resmgr_destroy(&sc->os_imgr);
}

/**
 * ice_apply_supported_speed_filter - Mask off unsupported speeds
 * @report_speeds: bit-field for the desired link speeds
 * @mod_type: type of module/sgmii connection we have
 *
 * Given a bitmap of the desired lenient mode link speeds,
 * this function will mask off the speeds that are not currently
 * supported by the device.
 */
static u16
ice_apply_supported_speed_filter(u16 report_speeds, u8 mod_type)
{
        u16 speed_mask;
        enum { IS_SGMII, IS_SFP, IS_QSFP } module;

        /*
         * The SFF specification says 0 is unknown, so we'll
         * treat it like we're connected through SGMII for now.
         * This may need revisiting if a new type is supported
         * in the future.
         */
        switch (mod_type) {
        case 0:
                module = IS_SGMII;
                break;
        case 3:
                module = IS_SFP;
                break;
        default:
                module = IS_QSFP;
                break;
        }

        /* We won't offer anything lower than 100M for any part,
         * but we'll need to mask off other speeds based on the
         * device and module type.
         */
        speed_mask = ~((u16)ICE_AQ_LINK_SPEED_100MB - 1);
        if ((report_speeds & ICE_AQ_LINK_SPEED_10GB) && (module == IS_SFP))
                speed_mask = ~((u16)ICE_AQ_LINK_SPEED_1000MB - 1);
        if (report_speeds & ICE_AQ_LINK_SPEED_25GB)
                speed_mask = ~((u16)ICE_AQ_LINK_SPEED_1000MB - 1);
        if (report_speeds & ICE_AQ_LINK_SPEED_50GB) {
                speed_mask = ~((u16)ICE_AQ_LINK_SPEED_1000MB - 1);
                if (module == IS_QSFP)
                        speed_mask = ~((u16)ICE_AQ_LINK_SPEED_10GB - 1);
        }
        if ((report_speeds & ICE_AQ_LINK_SPEED_100GB) ||
            (report_speeds & ICE_AQ_LINK_SPEED_200GB))
                speed_mask = ~((u16)ICE_AQ_LINK_SPEED_25GB - 1);
        return (report_speeds & speed_mask);
}

/**
 * ice_init_health_events - Enable FW health event reporting
 * @sc: device softc
 *
 * Will try to enable firmware health event reporting, but shouldn't
 * cause any grief (to the caller) if this fails.
 */
void
ice_init_health_events(struct ice_softc *sc)
{
        int status;
        u8 health_mask;

        if ((!ice_is_bit_set(sc->feat_cap, ICE_FEATURE_HEALTH_STATUS)) ||
            (!sc->enable_health_events))
                return;

        health_mask = ICE_AQC_HEALTH_STATUS_SET_PF_SPECIFIC_MASK |
                      ICE_AQC_HEALTH_STATUS_SET_GLOBAL_MASK;

        status = ice_aq_set_health_status_config(&sc->hw, health_mask, NULL);
        if (status)
                device_printf(sc->dev,
                    "Failed to enable firmware health events, err %s aq_err %s\n",
                    ice_status_str(status),
                    ice_aq_str(sc->hw.adminq.sq_last_status));
        else
                ice_set_bit(ICE_FEATURE_HEALTH_STATUS, sc->feat_en);
}

/**
 * ice_print_health_status_string - Print message for given FW health event
 * @dev: the PCIe device
 * @elem: health status element containing status code
 *
 * A rather large list of possible health status codes and their associated
 * messages.
 */
static void
ice_print_health_status_string(device_t dev,
                               struct ice_aqc_health_status_elem *elem)
{
        u16 status_code = le16toh(elem->health_status_code);

        switch (status_code) {
        case ICE_AQC_HEALTH_STATUS_INFO_RECOVERY:
                device_printf(dev, "The device is in firmware recovery mode.\n");
                device_printf(dev, "Possible Solution: Update to the latest NVM image.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_FLASH_ACCESS:
                device_printf(dev, "The flash chip cannot be accessed.\n");
                device_printf(dev, "Possible Solution: If issue persists, call customer support.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_NVM_AUTH:
                device_printf(dev, "NVM authentication failed.\n");
                device_printf(dev, "Possible Solution: Update to the latest NVM image.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_OROM_AUTH:
                device_printf(dev, "Option ROM authentication failed.\n");
                device_printf(dev, "Possible Solution: Update to the latest NVM image.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_DDP_AUTH:
                device_printf(dev, "DDP package failed.\n");
                device_printf(dev, "Possible Solution: Update to latest base driver and DDP package.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_NVM_COMPAT:
                device_printf(dev, "NVM image is incompatible.\n");
                device_printf(dev, "Possible Solution: Update to the latest NVM image.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_OROM_COMPAT:
                device_printf(dev, "Option ROM is incompatible.\n");
                device_printf(dev, "Possible Solution: Update to the latest NVM image.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_DCB_MIB:
                device_printf(dev, "Supplied MIB file is invalid. DCB reverted to default configuration.\n");
                device_printf(dev, "Possible Solution: Disable FW-LLDP and check DCBx system configuration.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_UNKNOWN_MOD_STRICT:
                device_printf(dev, "An unsupported module was detected.\n");
                device_printf(dev, "Possible Solution 1: Check your cable connection.\n");
                device_printf(dev, "Possible Solution 2: Change or replace the module or cable.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_MOD_TYPE:
                device_printf(dev, "Module type is not supported.\n");
                device_printf(dev, "Possible Solution: Change or replace the module or cable.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_MOD_QUAL:
                device_printf(dev, "Module is not qualified.\n");
                device_printf(dev, "Possible Solution 1: Check your cable connection.\n");
                device_printf(dev, "Possible Solution 2: Change or replace the module or cable.\n");
                device_printf(dev, "Possible Solution 3: Manually set speed and duplex.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_MOD_COMM:
                device_printf(dev, "Device cannot communicate with the module.\n");
                device_printf(dev, "Possible Solution 1: Check your cable connection.\n");
                device_printf(dev, "Possible Solution 2: Change or replace the module or cable.\n");
                device_printf(dev, "Possible Solution 3: Manually set speed and duplex.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_MOD_CONFLICT:
                device_printf(dev, "Unresolved module conflict.\n");
                device_printf(dev, "Possible Solution 1: Manually set speed/duplex or use Intel(R) Ethernet Port Configuration Tool to change the port option.\n");
                device_printf(dev, "Possible Solution 2: If the problem persists, use a cable/module that is found in the supported modules and cables list for this device.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_MOD_NOT_PRESENT:
                device_printf(dev, "Module is not present.\n");
                device_printf(dev, "Possible Solution 1: Check that the module is inserted correctly.\n");
                device_printf(dev, "Possible Solution 2: If the problem persists, use a cable/module that is found in the supported modules and cables list for this device.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_INFO_MOD_UNDERUTILIZED:
                device_printf(dev, "Underutilized module.\n");
                device_printf(dev, "Possible Solution 1: Change or replace the module or cable.\n");
                device_printf(dev, "Possible Solution 2: Use Intel(R) Ethernet Port Configuration Tool to change the port option.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_UNKNOWN_MOD_LENIENT:
                device_printf(dev, "An unsupported module was detected.\n");
                device_printf(dev, "Possible Solution 1: Check your cable connection.\n");
                device_printf(dev, "Possible Solution 2: Change or replace the module or cable.\n");
                device_printf(dev, "Possible Solution 3: Manually set speed and duplex.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_INVALID_LINK_CFG:
                device_printf(dev, "Invalid link configuration.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_PORT_ACCESS:
                device_printf(dev, "Port hardware access error.\n");
                device_printf(dev, "Possible Solution: Update to the latest NVM image.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_PORT_UNREACHABLE:
                device_printf(dev, "A port is unreachable.\n");
                device_printf(dev, "Possible Solution 1: Use Intel(R) Ethernet Port Configuration Tool to change the port option.\n");
                device_printf(dev, "Possible Solution 2: Update to the latest NVM image.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_INFO_PORT_SPEED_MOD_LIMITED:
                device_printf(dev, "Port speed is limited due to module.\n");
                device_printf(dev, "Possible Solution: Change the module or use Intel(R) Ethernet Port Configuration Tool to configure the port option to match the current module speed.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_PARALLEL_FAULT:
                device_printf(dev, "All configured link modes were attempted but failed to establish link.\n");
                device_printf(dev, "The device will restart the process to establish link.\n");
                device_printf(dev, "Possible Solution: Check link partner connection and configuration.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_INFO_PORT_SPEED_PHY_LIMITED:
                device_printf(dev, "Port speed is limited by PHY capabilities.\n");
                device_printf(dev, "Possible Solution 1: Change the module to align to port option.\n");
                device_printf(dev, "Possible Solution 2: Use Intel(R) Ethernet Port Configuration Tool to change the port option.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_NETLIST_TOPO:
                device_printf(dev, "LOM topology netlist is corrupted.\n");
                device_printf(dev, "Possible Solution: Update to the latest NVM image.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_NETLIST:
                device_printf(dev, "Unrecoverable netlist error.\n");
                device_printf(dev, "Possible Solution: Update to the latest NVM image.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_TOPO_CONFLICT:
                device_printf(dev, "Port topology conflict.\n");
                device_printf(dev, "Possible Solution 1: Use Intel(R) Ethernet Port Configuration Tool to change the port option.\n");
                device_printf(dev, "Possible Solution 2: Update to the latest NVM image.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_LINK_HW_ACCESS:
                device_printf(dev, "Unrecoverable hardware access error.\n");
                device_printf(dev, "Possible Solution: Update to the latest NVM image.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_LINK_RUNTIME:
                device_printf(dev, "Unrecoverable runtime error.\n");
                device_printf(dev, "Possible Solution: Update to the latest NVM image.\n");
                break;
        case ICE_AQC_HEALTH_STATUS_ERR_DNL_INIT:
                device_printf(dev, "Link management engine failed to initialize.\n");
                device_printf(dev, "Possible Solution: Update to the latest NVM image.\n");
                break;
        default:
                break;
        }
}

/**
 * ice_handle_health_status_event - helper function to output health status
 * @sc: device softc structure
 * @event: event received on a control queue
 *
 * Prints out the appropriate string based on the given Health Status Event
 * code.
 */
static void
ice_handle_health_status_event(struct ice_softc *sc,
                               struct ice_rq_event_info *event)
{
        struct ice_aqc_health_status_elem *health_info;
        u16 status_count;
        int i;

        if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_HEALTH_STATUS))
                return;

        health_info = (struct ice_aqc_health_status_elem *)event->msg_buf;
        status_count = le16toh(event->desc.params.get_health_status.health_status_count);

        if (status_count > (event->buf_len / sizeof(*health_info))) {
                device_printf(sc->dev, "Received a health status event with invalid event count\n");
                return;
        }

        for (i = 0; i < status_count; i++) {
                ice_print_health_status_string(sc->dev, health_info);
                health_info++;
        }
}

/**
 * ice_set_default_local_lldp_mib - Possibly apply local LLDP MIB to FW
 * @sc: device softc structure
 *
 * This function needs to be called after link up; it makes sure the FW has
 * certain PFC/DCB settings. In certain configurations this will re-apply a
 * default local LLDP MIB configuration; this is intended to workaround a FW
 * behavior where these settings seem to be cleared on link up.
 */
void
ice_set_default_local_lldp_mib(struct ice_softc *sc)
{
        struct ice_hw *hw = &sc->hw;
        struct ice_port_info *pi;
        device_t dev = sc->dev;
        int status;

        /* Set Local MIB can disrupt flow control settings for
         * non-DCB-supported devices.
         */
        if (!ice_is_bit_set(sc->feat_en, ICE_FEATURE_DCB))
                return;

        pi = hw->port_info;

        /* Don't overwrite a custom SW configuration */
        if (!pi->qos_cfg.is_sw_lldp &&
            !ice_test_state(&sc->state, ICE_STATE_MULTIPLE_TCS))
                ice_set_default_local_mib_settings(sc);

        status = ice_set_dcb_cfg(pi);

        if (status)
                device_printf(dev,
                    "Error setting Local LLDP MIB: %s aq_err %s\n",
                    ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
}

/**
 * ice_sbuf_print_ets_cfg - Helper function to print ETS cfg
 * @sbuf: string buffer to print to
 * @name: prefix string to use
 * @ets: structure to pull values from
 *
 * A helper function for ice_sysctl_dump_dcbx_cfg(), this
 * formats the ETS rec and cfg TLVs into text.
 */
static void
ice_sbuf_print_ets_cfg(struct sbuf *sbuf, const char *name, struct ice_dcb_ets_cfg *ets)
{
        sbuf_printf(sbuf, "%s.willing: %u\n", name, ets->willing);
        sbuf_printf(sbuf, "%s.cbs: %u\n", name, ets->cbs);
        sbuf_printf(sbuf, "%s.maxtcs: %u\n", name, ets->maxtcs);

        sbuf_printf(sbuf, "%s.prio_table:", name);
        for (int i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++)
                sbuf_printf(sbuf, " %d", ets->prio_table[i]);
        sbuf_printf(sbuf, "\n");

        sbuf_printf(sbuf, "%s.tcbwtable:", name);
        for (int i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++)
                sbuf_printf(sbuf, " %d", ets->tcbwtable[i]);
        sbuf_printf(sbuf, "\n");

        sbuf_printf(sbuf, "%s.tsatable:", name);
        for (int i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++)
                sbuf_printf(sbuf, " %d", ets->tsatable[i]);
        sbuf_printf(sbuf, "\n");
}

/**
 * ice_sysctl_dump_dcbx_cfg - Print out DCBX/DCB config info
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: AQ define for either Local or Remote MIB
 * @req: sysctl request pointer
 *
 * Prints out DCB/DCBX configuration, including the contents
 * of either the local or remote MIB, depending on the value
 * used in arg2.
 */
static int
ice_sysctl_dump_dcbx_cfg(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_aqc_get_cee_dcb_cfg_resp cee_cfg = {};
        struct ice_dcbx_cfg dcb_buf = {};
        struct ice_dcbx_cfg *dcbcfg;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        struct sbuf *sbuf;
        int status;
        u8 maxtcs, dcbx_status, is_sw_lldp;

        UNREFERENCED_PARAMETER(oidp);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        is_sw_lldp = hw->port_info->qos_cfg.is_sw_lldp;

        /* The driver doesn't receive a Remote MIB via SW */
        if (is_sw_lldp && arg2 == ICE_AQ_LLDP_MIB_REMOTE)
                return (ENOENT);

        dcbcfg = &hw->port_info->qos_cfg.local_dcbx_cfg;
        if (!is_sw_lldp) {
                /* Collect information from the FW in FW LLDP mode */
                dcbcfg = &dcb_buf;
                status = ice_aq_get_dcb_cfg(hw, (u8)arg2,
                    ICE_AQ_LLDP_BRID_TYPE_NEAREST_BRID, dcbcfg);
                if (status && arg2 == ICE_AQ_LLDP_MIB_REMOTE &&
                    hw->adminq.sq_last_status == ICE_AQ_RC_ENOENT) {
                        device_printf(dev,
                            "Unable to query Remote MIB; port has not received one yet\n");
                        return (ENOENT);
                }
                if (status) {
                        device_printf(dev, "Unable to query LLDP MIB, err %s aq_err %s\n",
                            ice_status_str(status),
                            ice_aq_str(hw->adminq.sq_last_status));
                        return (EIO);
                }
        }

        status = ice_aq_get_cee_dcb_cfg(hw, &cee_cfg, NULL);
        if (!status)
                dcbcfg->dcbx_mode = ICE_DCBX_MODE_CEE;
        else if (hw->adminq.sq_last_status == ICE_AQ_RC_ENOENT)
                dcbcfg->dcbx_mode = ICE_DCBX_MODE_IEEE;
        else
                device_printf(dev, "Get CEE DCB Cfg AQ cmd err %s aq_err %s\n",
                    ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));

        maxtcs = hw->func_caps.common_cap.maxtc;
        dcbx_status = ice_get_dcbx_status(hw);

        sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);

        /* Do the actual printing */
        sbuf_printf(sbuf, "\n");
        sbuf_printf(sbuf, "SW LLDP mode: %d\n", is_sw_lldp);
        sbuf_printf(sbuf, "Function caps maxtcs: %d\n", maxtcs);
        sbuf_printf(sbuf, "dcbx_status: %d\n", dcbx_status);

        sbuf_printf(sbuf, "numapps: %u\n", dcbcfg->numapps);
        sbuf_printf(sbuf, "CEE TLV status: %u\n", dcbcfg->tlv_status);
        sbuf_printf(sbuf, "pfc_mode: %s\n", (dcbcfg->pfc_mode == ICE_QOS_MODE_DSCP) ?
            "DSCP" : "VLAN");
        sbuf_printf(sbuf, "dcbx_mode: %s\n",
            (dcbcfg->dcbx_mode == ICE_DCBX_MODE_IEEE) ? "IEEE" :
            (dcbcfg->dcbx_mode == ICE_DCBX_MODE_CEE) ? "CEE" :
            "Unknown");

        ice_sbuf_print_ets_cfg(sbuf, "etscfg", &dcbcfg->etscfg);
        ice_sbuf_print_ets_cfg(sbuf, "etsrec", &dcbcfg->etsrec);

        sbuf_printf(sbuf, "pfc.willing: %u\n", dcbcfg->pfc.willing);
        sbuf_printf(sbuf, "pfc.mbc: %u\n", dcbcfg->pfc.mbc);
        sbuf_printf(sbuf, "pfc.pfccap: 0x%0x\n", dcbcfg->pfc.pfccap);
        sbuf_printf(sbuf, "pfc.pfcena: 0x%0x\n", dcbcfg->pfc.pfcena);

        if (arg2 == ICE_AQ_LLDP_MIB_LOCAL) {
                sbuf_printf(sbuf, "dscp_map:\n");
                for (int i = 0; i < 8; i++) {
                        for (int j = 0; j < 8; j++)
                                sbuf_printf(sbuf, " %d",
                                            dcbcfg->dscp_map[i * 8 + j]);
                        sbuf_printf(sbuf, "\n");
                }

                sbuf_printf(sbuf, "\nLocal registers:\n");
                sbuf_printf(sbuf, "PRTDCB_GENC.NUMTC: %d\n",
                    (rd32(hw, PRTDCB_GENC) & PRTDCB_GENC_NUMTC_M)
                        >> PRTDCB_GENC_NUMTC_S);
                sbuf_printf(sbuf, "PRTDCB_TUP2TC: 0x%0x\n",
                    (rd32(hw, PRTDCB_TUP2TC)));
                sbuf_printf(sbuf, "PRTDCB_RUP2TC: 0x%0x\n",
                    (rd32(hw, PRTDCB_RUP2TC)));
                sbuf_printf(sbuf, "GLDCB_TC2PFC: 0x%0x\n",
                    (rd32(hw, GLDCB_TC2PFC)));
        }

        /* Finish */
        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        return (0);
}

/**
 * ice_sysctl_dump_vsi_cfg - print PF LAN VSI configuration
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * XXX: This could be extended to apply to arbitrary PF-owned VSIs,
 * but for simplicity, this only works on the PF's LAN VSI.
 */
static int
ice_sysctl_dump_vsi_cfg(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_vsi_ctx ctx = { 0 };
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        struct sbuf *sbuf;
        int status;

        UNREFERENCED_PARAMETER(oidp);
        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        /* Get HW absolute index of a VSI */
        ctx.vsi_num = ice_get_hw_vsi_num(hw, sc->pf_vsi.idx);

        status = ice_aq_get_vsi_params(hw, &ctx, NULL);
        if (status) {
                device_printf(dev,
                    "Get VSI AQ call failed, err %s aq_err %s\n",
                    ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);

        /* Do the actual printing */
        sbuf_printf(sbuf, "\n");

        sbuf_printf(sbuf, "VSI NUM: %d\n", ctx.vsi_num);
        sbuf_printf(sbuf, "VF  NUM: %d\n", ctx.vf_num);
        sbuf_printf(sbuf, "VSIs allocated: %d\n", ctx.vsis_allocd);
        sbuf_printf(sbuf, "VSIs unallocated: %d\n", ctx.vsis_unallocated);

        sbuf_printf(sbuf, "Rx Queue Map method: %d\n",
            LE16_TO_CPU(ctx.info.mapping_flags));
        /* The PF VSI is always contiguous, so there's no if-statement here */
        sbuf_printf(sbuf, "Rx Queue base: %d\n",
            LE16_TO_CPU(ctx.info.q_mapping[0]));
        sbuf_printf(sbuf, "Rx Queue count: %d\n",
            LE16_TO_CPU(ctx.info.q_mapping[1]));

        sbuf_printf(sbuf, "TC qbases  :");
        for (int i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++) {
                sbuf_printf(sbuf, " %4d",
                    ctx.info.tc_mapping[i] & ICE_AQ_VSI_TC_Q_OFFSET_M);
        }
        sbuf_printf(sbuf, "\n");

        sbuf_printf(sbuf, "TC qcounts :");
        for (int i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++) {
                sbuf_printf(sbuf, " %4d",
                    1 << (ctx.info.tc_mapping[i] >> ICE_AQ_VSI_TC_Q_NUM_S));
        }

        /* Finish */
        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        return (0);
}

/**
 * ice_get_tx_rx_equalizations -- read serdes tx rx equalization params
 * @hw: pointer to the HW struct
 * @serdes_num: represents the serdes number
 * @ptr: structure to read all serdes parameter for given serdes
 *
 * returns all serdes equalization parameter supported per serdes number
 */
static int
ice_get_tx_rx_equalizations(struct ice_hw *hw, u8 serdes_num,
                            struct ice_serdes_equalization *ptr)
{
        int err = 0;

        if (!ptr)
                return (EOPNOTSUPP);

#define ICE_GET_PHY_EQUALIZATION(equ, dir, value) \
        ice_aq_get_phy_equalization(hw, equ, dir, serdes_num, &(ptr->value))

        err = ICE_GET_PHY_EQUALIZATION(ICE_AQC_RX_EQU_PRE1,
                        ICE_AQC_OP_CODE_RX_EQU, rx_equalization_pre1);
        if (err)
                return err;

        err = ICE_GET_PHY_EQUALIZATION(ICE_AQC_RX_EQU_PRE2,
                        ICE_AQC_OP_CODE_RX_EQU, rx_equalization_pre2);
        if (err)
                return err;

        err = ICE_GET_PHY_EQUALIZATION(ICE_AQC_RX_EQU_POST1,
                        ICE_AQC_OP_CODE_RX_EQU, rx_equalization_post1);
        if (err)
                return err;

        err = ICE_GET_PHY_EQUALIZATION(ICE_AQC_RX_EQU_BFLF,
                        ICE_AQC_OP_CODE_RX_EQU, rx_equalization_bflf);
        if (err)
                return err;

        err = ICE_GET_PHY_EQUALIZATION(ICE_AQC_RX_EQU_BFHF,
                        ICE_AQC_OP_CODE_RX_EQU, rx_equalization_bfhf);
        if (err)
                return err;

        err = ICE_GET_PHY_EQUALIZATION(ICE_AQC_RX_EQU_DRATE,
                        ICE_AQC_OP_CODE_RX_EQU, rx_equalization_drate);
        if (err)
                return err;

        err = ICE_GET_PHY_EQUALIZATION(ICE_AQC_TX_EQU_PRE1,
                        ICE_AQC_OP_CODE_TX_EQU, tx_equalization_pre1);
        if (err)
                return err;

        err = ICE_GET_PHY_EQUALIZATION(ICE_AQC_TX_EQU_PRE2,
                        ICE_AQC_OP_CODE_TX_EQU, tx_equalization_pre2);
        if (err)
                return err;

        err = ICE_GET_PHY_EQUALIZATION(ICE_AQC_TX_EQU_PRE3,
                        ICE_AQC_OP_CODE_TX_EQU, tx_equalization_pre3);
        if (err)
                return err;

        err = ICE_GET_PHY_EQUALIZATION(ICE_AQC_TX_EQU_ATTEN,
                        ICE_AQC_OP_CODE_TX_EQU, tx_equalization_atten);
        if (err)
                return err;

        err = ICE_GET_PHY_EQUALIZATION(ICE_AQC_TX_EQU_POST1,
                        ICE_AQC_OP_CODE_TX_EQU, tx_equalization_post1);
        if (err)
                return err;

        return (0);
}

/**
 * ice_fec_counter_read - reads FEC stats from PHY
 * @hw: pointer to the HW struct
 * @receiver_id: pcsquad at registerlevel
 * @reg_offset: register for the current request
 * @output: pointer to the caller-supplied buffer to return requested fec stats
 *
 * Returns fec stats from phy
 */
static int
ice_fec_counter_read(struct ice_hw *hw, u32 receiver_id, u32 reg_offset,
                            u16 *output)
{
        u16 flag = (ICE_AQ_FLAG_RD | ICE_AQ_FLAG_BUF | ICE_AQ_FLAG_SI);
        struct ice_sbq_msg_input msg = {};
        int err = 0;

        memset(&msg, 0, sizeof(msg));
        msg.msg_addr_low = ICE_LO_WORD(reg_offset);
        msg.msg_addr_high = ICE_LO_DWORD(receiver_id);
        msg.opcode = ice_sbq_msg_rd;
        msg.dest_dev = rmn_0;

        err = ice_sbq_rw_reg(hw, &msg, flag);
        if (err) {
                return err;
        }
        *output = ICE_LO_WORD(msg.data);
        return (0);
}

/**
 * ice_get_port_fec_stats - returns fec correctable, uncorrectable stats per pcsquad, pcsport
 * @hw: pointer to the HW struct
 * @pcs_quad: pcsquad for input port
 * @pcs_port: pcsport for input port
 * @fec_stats: buffer to hold fec statistics for given port
 *
 * Returns fec stats
 */
static int
ice_get_port_fec_stats(struct ice_hw *hw, u16 pcs_quad, u16 pcs_port,
                       struct ice_fec_stats_to_sysctl *fec_stats)
{
        u32 uncorr_low_reg = 0, uncorr_high_reg = 0;
        u16 uncorr_low_val = 0, uncorr_high_val = 0;
        u32 corr_low_reg = 0, corr_high_reg = 0;
        u16 corr_low_val = 0, corr_high_val = 0;
        u32 receiver_id = 0;
        int err;

        switch (pcs_port) {
        case 0:
                corr_low_reg = ICE_RS_FEC_CORR_LOW_REG_PORT0;
                corr_high_reg = ICE_RS_FEC_CORR_HIGH_REG_PORT0;
                uncorr_low_reg = ICE_RS_FEC_UNCORR_LOW_REG_PORT0;
                uncorr_high_reg = ICE_RS_FEC_UNCORR_HIGH_REG_PORT0;
                break;
        case 1:
                corr_low_reg = ICE_RS_FEC_CORR_LOW_REG_PORT1;
                corr_high_reg = ICE_RS_FEC_CORR_HIGH_REG_PORT1;
                uncorr_low_reg = ICE_RS_FEC_UNCORR_LOW_REG_PORT1;
                uncorr_high_reg = ICE_RS_FEC_UNCORR_HIGH_REG_PORT1;
                break;
        case 2:
                corr_low_reg = ICE_RS_FEC_CORR_LOW_REG_PORT2;
                corr_high_reg = ICE_RS_FEC_CORR_HIGH_REG_PORT2;
                uncorr_low_reg = ICE_RS_FEC_UNCORR_LOW_REG_PORT2;
                uncorr_high_reg = ICE_RS_FEC_UNCORR_HIGH_REG_PORT2;
                break;
        case 3:
                corr_low_reg = ICE_RS_FEC_CORR_LOW_REG_PORT3;
                corr_high_reg = ICE_RS_FEC_CORR_HIGH_REG_PORT3;
                uncorr_low_reg = ICE_RS_FEC_UNCORR_LOW_REG_PORT3;
                uncorr_high_reg = ICE_RS_FEC_UNCORR_HIGH_REG_PORT3;
                break;
        default:
                return (EINVAL);
        }
        if (pcs_quad == 0)
                receiver_id = ICE_RS_FEC_RECEIVER_ID_PCS0; /* MTIP PCS Quad 0 -FEC */
        else if (pcs_quad == 1)
                receiver_id = ICE_RS_FEC_RECEIVER_ID_PCS1; /* MTIP PCS Quad 1 -FEC */
        else
                return (EINVAL);

        err = ice_fec_counter_read(hw, receiver_id, corr_low_reg,
                        &corr_low_val);
        if (err)
                return err;

        err = ice_fec_counter_read(hw, receiver_id, corr_high_reg,
                        &corr_high_val);
        if (err)
                return err;

        err = ice_fec_counter_read(hw, receiver_id, uncorr_low_reg,
                        &uncorr_low_val);
        if (err)
                return err;

        err = ice_fec_counter_read(hw, receiver_id, uncorr_high_reg,
                        &uncorr_high_val);
        if (err)
                return err;

        fec_stats->fec_corr_cnt_low =  corr_low_val;
        fec_stats->fec_corr_cnt_high =  corr_high_val;
        fec_stats->fec_uncorr_cnt_low =  uncorr_low_val;
        fec_stats->fec_uncorr_cnt_high =  uncorr_high_val;

        return (0);
}

/**
 * ice_is_serdes_muxed - returns whether serdes is muxed in hardware
 * @hw: pointer to the HW struct
 *
 * Returns True : when serdes is muxed
 *         False: when serdes is not muxed
 */
static bool
ice_is_serdes_muxed(struct ice_hw *hw)
{
        return (rd32(hw, 0xB81E0) & 0x4);
}

/**
 * ice_get_maxspeed - Get the max speed for given lport
 * @hw: pointer to the HW struct
 * @lport: logical port for which max speed is requested
 * @max_speed: return max speed for input lport
 */
static int
ice_get_maxspeed(struct ice_hw *hw, u8 lport, u8 *max_speed)
{
        struct ice_aqc_get_port_options_elem options[ICE_AQC_PORT_OPT_MAX] = {};
        u8 option_count = ICE_AQC_PORT_OPT_MAX;
        bool active_valid, pending_valid;
        u8 active_idx, pending_idx;
        int status;

        status = ice_aq_get_port_options(hw, options, &option_count,
                        lport, true, &active_idx, &active_valid,
                        &pending_idx, &pending_valid);

        if (status || active_idx >= ICE_AQC_PORT_OPT_MAX) {
                ice_debug(hw, ICE_DBG_PHY, "Port split read err: %d\n", status);
                return (EIO);
        }

        if (active_valid) {
                ice_debug(hw, ICE_DBG_PHY, "Active idx: %d\n", active_idx);
        } else {
                ice_debug(hw, ICE_DBG_PHY, "No valid Active option\n");
                return (EINVAL);
        }
        *max_speed = options[active_idx].max_lane_speed;

        return (0);
}

/**
 * ice_update_port_topology - update port topology
 * @lport: logical port for which physical info requested
 * @port_topology: buffer to hold port topology
 * @is_muxed: serdes is muxed in hardware
 */
static int
ice_update_port_topology(u8 lport, struct ice_port_topology *port_topology,
                bool is_muxed)
{
        switch (lport) {
        case 0:
                port_topology->pcs_quad_select = 0;
                port_topology->pcs_port = 0;
                port_topology->primary_serdes_lane = 0;
                break;
        case 1:
                port_topology->pcs_quad_select = 1;
                port_topology->pcs_port = 0;
                if (is_muxed == true)
                        port_topology->primary_serdes_lane = 2;
                else
                        port_topology->primary_serdes_lane = 4;
                break;
        case 2:
                port_topology->pcs_quad_select = 0;
                port_topology->pcs_port = 1;
                port_topology->primary_serdes_lane = 1;
                break;
        case 3:
                port_topology->pcs_quad_select = 1;
                port_topology->pcs_port = 1;
                if (is_muxed == true)
                        port_topology->primary_serdes_lane = 3;
                else
                        port_topology->primary_serdes_lane = 5;
                break;
        case 4:
                port_topology->pcs_quad_select = 0;
                port_topology->pcs_port = 2;
                port_topology->primary_serdes_lane = 2;
                break;
        case 5:
                port_topology->pcs_quad_select = 1;
                port_topology->pcs_port = 2;
                port_topology->primary_serdes_lane = 6;
                break;
        case 6:
                port_topology->pcs_quad_select = 0;
                port_topology->pcs_port = 3;
                port_topology->primary_serdes_lane = 3;
                break;
        case 7:
                port_topology->pcs_quad_select = 1;
                port_topology->pcs_port = 3;
                port_topology->primary_serdes_lane = 7;
                break;
        default:
                return (EINVAL);
        }
        return 0;
}

/**
 * ice_get_port_topology - returns physical topology
 * @hw: pointer to the HW struct
 * @lport: logical port for which physical info requested
 * @port_topology: buffer to hold port topology
 *
 * Returns the physical component associated with the Port like pcsquad, pcsport, serdesnumber
 */
static int
ice_get_port_topology(struct ice_hw *hw, u8 lport,
                      struct ice_port_topology *port_topology)
{
        struct ice_aqc_get_link_topo cmd;
        bool is_muxed = false;
        u8 cage_type = 0;
        u16 node_handle;
        u8 ctx = 0;
        int err;

        if (!hw || !port_topology)
                return (EINVAL);

        if (hw->device_id >= ICE_DEV_ID_E810_XXV_BACKPLANE) {
                port_topology->serdes_lane_count = 1;
                if (lport == 0) {
                        port_topology->pcs_quad_select = 0;
                        port_topology->pcs_port = 0;
                        port_topology->primary_serdes_lane = 0;
                } else if (lport == 1) {
                        port_topology->pcs_quad_select = 1;
                        port_topology->pcs_port = 0;
                        port_topology->primary_serdes_lane = 1;
                } else {
                        return (EINVAL);
                }
                return (0);
        }

        memset(&cmd, 0, sizeof(cmd));
        ctx = ICE_AQC_LINK_TOPO_NODE_TYPE_CAGE << ICE_AQC_LINK_TOPO_NODE_TYPE_S;
        ctx |= ICE_AQC_LINK_TOPO_NODE_CTX_PORT << ICE_AQC_LINK_TOPO_NODE_CTX_S;
        cmd.addr.topo_params.node_type_ctx = ctx;
        cmd.addr.topo_params.index = 0;
        cmd.addr.topo_params.lport_num = 0;
        cmd.addr.topo_params.lport_num_valid = 0;

        err = ice_aq_get_netlist_node(hw, &cmd, &cage_type, &node_handle);
        if (err)
                return (EINVAL);

        is_muxed = ice_is_serdes_muxed(hw);

        err = ice_update_port_topology(lport, port_topology, is_muxed);
        if (err)
                return err;

        if (cage_type == 0x11 ||  /* SFP */
            cage_type == 0x12) {   /* SFP28 */
                port_topology->serdes_lane_count = 1;
        } else if (cage_type == 0x13 ||  /* QSFP */
            cage_type == 0x14) {   /* QSFP28 */
                u8 max_speed = 0;

                err = ice_get_maxspeed(hw, port_topology->primary_serdes_lane,
                    &max_speed);
                if (err)
                        return err;

                if (max_speed == ICE_AQC_PORT_OPT_MAX_LANE_M)
                        device_printf(ice_hw_to_dev(hw),
                            "%s: WARNING: reported max_lane_speed is N/A\n",
                            __func__);

                if (max_speed == ICE_AQC_PORT_OPT_MAX_LANE_100G)
                        port_topology->serdes_lane_count = 4;
                else if (max_speed == ICE_AQC_PORT_OPT_MAX_LANE_50G ||
                    max_speed == ICE_AQC_PORT_OPT_MAX_LANE_40G)
                        port_topology->serdes_lane_count = 2;
                else
                        port_topology->serdes_lane_count = 1;
        } else
                return (EINVAL);

        ice_debug(hw, ICE_DBG_PHY, "%s: Port Topology (lport %d):\n",
            __func__, lport);
        ice_debug(hw, ICE_DBG_PHY, "serdes lane count %d\n",
            port_topology->serdes_lane_count);
        ice_debug(hw, ICE_DBG_PHY, "pcs quad select %d\n",
            port_topology->pcs_quad_select);
        ice_debug(hw, ICE_DBG_PHY, "pcs port %d\n",
            port_topology->pcs_port);
        ice_debug(hw, ICE_DBG_PHY, "primary serdes lane %d\n",
            port_topology->primary_serdes_lane);

        return (0);
}

/**
 * ice_sysctl_dump_phy_stats - print PHY stats
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 */
static int
ice_sysctl_dump_phy_stats(SYSCTL_HANDLER_ARGS)
{
        struct ice_regdump_to_sysctl ice_prv_regs_buf = {};
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_port_topology port_topology;
        struct ice_hw *hw = &sc->hw;
        struct ice_port_info *pi;
        device_t dev = sc->dev;
        u8 serdes_num = 0;
        unsigned int i;
        int err = 0;
        struct sbuf *sbuf;

        pi = hw->port_info;

        if (!pi) {
                device_printf(dev, "Port info structure is null\n");
                return (EINVAL);
        }

        UNREFERENCED_PARAMETER(oidp);
        UNREFERENCED_PARAMETER(arg2);
        UNREFERENCED_PARAMETER(req);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        if (ice_get_port_topology(hw, pi->lport, &port_topology) != 0) {
                device_printf(dev,
                              "Extended register dump failed for Lport %d\n",
                              pi->lport);
                return (EIO);
        }

        if (port_topology.serdes_lane_count > ICE_MAX_SERDES_LANE_COUNT) {
                device_printf(dev,
                        "Extended register dump failed: Lport %d Serdes count %d\n",
                        pi->lport,
                        port_topology.serdes_lane_count);
                return (EINVAL);
        }

        sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);
        /* Get serdes equalization parameter for available serdes */
        for (i = 0; i < port_topology.serdes_lane_count; i++) {
                serdes_num = port_topology.primary_serdes_lane + i;
                err = ice_get_tx_rx_equalizations(hw, serdes_num,
                                &(ice_prv_regs_buf.equalization[i]));
                if (err) {
                        device_printf(dev,
                            "Serdes equalization get failed Lport %d Serdes %d Err %d\n",
                            pi->lport,serdes_num, err);
                        sbuf_finish(sbuf);
                        sbuf_delete(sbuf);
                        return (EIO);
                }
                sbuf_printf(sbuf, "\nSerdes lane: %d\n", i);
                sbuf_printf(sbuf, "RX PRE1 = %d\n",
                        ice_prv_regs_buf.equalization[i].rx_equalization_pre1);
                sbuf_printf(sbuf, "RX PRE2 = %d\n",
                        (s16)ice_prv_regs_buf.equalization[i].rx_equalization_pre2);
                sbuf_printf(sbuf, "RX POST1 = %d\n",
                        ice_prv_regs_buf.equalization[i].rx_equalization_post1);
                sbuf_printf(sbuf, "RX BFLF = %d\n",
                        ice_prv_regs_buf.equalization[i].rx_equalization_bflf);
                sbuf_printf(sbuf, "RX BFHF = %d\n",
                        ice_prv_regs_buf.equalization[i].rx_equalization_bfhf);
                sbuf_printf(sbuf, "RX DRATE = %d\n",
                        (s16)ice_prv_regs_buf.equalization[i].rx_equalization_drate);
                sbuf_printf(sbuf, "TX PRE1 = %d\n",
                        ice_prv_regs_buf.equalization[i].tx_equalization_pre1);
                sbuf_printf(sbuf, "TX PRE2 = %d\n",
                        ice_prv_regs_buf.equalization[i].tx_equalization_pre2);
                sbuf_printf(sbuf, "TX PRE3 = %d\n",
                        ice_prv_regs_buf.equalization[i].tx_equalization_pre3);
                sbuf_printf(sbuf, "TX POST1 = %d\n",
                        ice_prv_regs_buf.equalization[i].tx_equalization_post1);
                sbuf_printf(sbuf, "TX ATTEN = %d\n",
                        ice_prv_regs_buf.equalization[i].tx_equalization_atten);
        }

        /* Get fec  correctable , uncorrectable counter */
        err = ice_get_port_fec_stats(hw, port_topology.pcs_quad_select,
                                     port_topology.pcs_port,
                                     &(ice_prv_regs_buf.stats));
        if (err) {
                device_printf(dev, "failed to get FEC stats Lport %d Err %d\n",
                                pi->lport, err);
                sbuf_finish(sbuf);
                sbuf_delete(sbuf);
                return (EIO);
        }

        sbuf_printf(sbuf, "\nRS FEC Corrected codeword count = %d\n",
                        ((u32)ice_prv_regs_buf.stats.fec_corr_cnt_high << 16) |
                            ice_prv_regs_buf.stats.fec_corr_cnt_low);
        sbuf_printf(sbuf, "RS FEC Uncorrected codeword count = %d\n",
                        ((u32)ice_prv_regs_buf.stats.fec_uncorr_cnt_high << 16) |
                            ice_prv_regs_buf.stats.fec_uncorr_cnt_low);

        /* Finish */
        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        return (0);
}

/**
 * ice_ets_str_to_tbl - Parse string into ETS table
 * @str: input string to parse
 * @table: output eight values used for ETS values
 * @limit: max valid value to accept for ETS values
 *
 * Parses a string and converts the eight values within
 * into a table that can be used in setting ETS settings
 * in a MIB.
 *
 * @return 0 on success, EINVAL if a parsed value is
 * not between 0 and limit.
 */
static int
ice_ets_str_to_tbl(const char *str, u8 *table, u8 limit)
{
        const char *str_start = str;
        char *str_end;
        long token;

        for (int i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++) {
                token = strtol(str_start, &str_end, 0);
                if (token < 0 || token > limit)
                        return (EINVAL);

                table[i] = (u8)token;
                str_start = (str_end + 1);
        }

        return (0);
}

/**
 * ice_check_ets_bw - Check if ETS bw vals are valid
 * @table: eight values used for ETS bandwidth
 *
 * @return true if the sum of all 8 values in table
 * equals 100.
 */
static bool
ice_check_ets_bw(u8 *table)
{
        int sum = 0;
        for (int i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++)
                sum += (int)table[i];

        return (sum == 100);
}

/**
 * ice_cfg_pba_num - Determine if PBA Number is retrievable
 * @sc: the device private softc structure
 *
 * Sets the feature flag for the existence of a PBA number
 * based on the success of the read command.  This does not
 * cache the result.
 */
void
ice_cfg_pba_num(struct ice_softc *sc)
{
        u8 pba_string[32] = "";

        if ((ice_is_bit_set(sc->feat_cap, ICE_FEATURE_HAS_PBA)) &&
            (ice_read_pba_string(&sc->hw, pba_string, sizeof(pba_string)) == 0))
                ice_set_bit(ICE_FEATURE_HAS_PBA, sc->feat_en);
}

/**
 * ice_sysctl_query_port_ets - print Port ETS Config from AQ
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 */
static int
ice_sysctl_query_port_ets(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_aqc_port_ets_elem port_ets = { 0 };
        struct ice_hw *hw = &sc->hw;
        struct ice_port_info *pi;
        device_t dev = sc->dev;
        struct sbuf *sbuf;
        int status;
        int i = 0;

        UNREFERENCED_PARAMETER(oidp);
        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        pi = hw->port_info;

        status = ice_aq_query_port_ets(pi, &port_ets, sizeof(port_ets), NULL);
        if (status) {
                device_printf(dev,
                    "Query Port ETS AQ call failed, err %s aq_err %s\n",
                    ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        sbuf = sbuf_new_for_sysctl(NULL, NULL, 128, req);

        /* Do the actual printing */
        sbuf_printf(sbuf, "\n");

        sbuf_printf(sbuf, "Valid TC map: 0x%x\n", port_ets.tc_valid_bits);

        sbuf_printf(sbuf, "TC BW %%:");
        ice_for_each_traffic_class(i) {
                sbuf_printf(sbuf, " %3d", port_ets.tc_bw_share[i]);
        }
        sbuf_printf(sbuf, "\n");

        sbuf_printf(sbuf, "EIR profile ID: %d\n", port_ets.port_eir_prof_id);
        sbuf_printf(sbuf, "CIR profile ID: %d\n", port_ets.port_cir_prof_id);
        sbuf_printf(sbuf, "TC Node prio: 0x%x\n", port_ets.tc_node_prio);

        sbuf_printf(sbuf, "TC Node TEIDs:\n");
        ice_for_each_traffic_class(i) {
                sbuf_printf(sbuf, "%d: %d\n", i, port_ets.tc_node_teid[i]);
        }

        /* Finish */
        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        return (0);
}

/**
 * ice_sysctl_dscp2tc_map - Map DSCP to hardware TCs
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: which eight DSCP to UP mappings to configure (0 - 7)
 * @req: sysctl request pointer
 *
 * Gets or sets the current DSCP to UP table cached by the driver. Since there
 * are 64 possible DSCP values to configure, this sysctl only configures
 * chunks of 8 in that space at a time.
 *
 * This sysctl is only relevant in DSCP mode, and will only function in SW DCB
 * mode.
 */
static int
ice_sysctl_dscp2tc_map(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_dcbx_cfg *local_dcbx_cfg;
        struct ice_port_info *pi;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;
        struct sbuf *sbuf;
        int ret;

        /* Store input rates from user */
        char dscp_user_buf[128] = "";
        u8 new_dscp_table_seg[ICE_MAX_TRAFFIC_CLASS] = {};

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        if (req->oldptr == NULL && req->newptr == NULL) {
                ret = SYSCTL_OUT(req, 0, 128);
                return (ret);
        }

        pi = hw->port_info;
        local_dcbx_cfg = &pi->qos_cfg.local_dcbx_cfg;

        sbuf = sbuf_new(NULL, dscp_user_buf, 128, SBUF_FIXEDLEN | SBUF_INCLUDENUL);

        /* Format DSCP-to-UP data for output */
        for (int i = 0; i < ICE_MAX_TRAFFIC_CLASS; i++) {
                sbuf_printf(sbuf, "%d", local_dcbx_cfg->dscp_map[arg2 * 8 + i]);
                if (i != ICE_MAX_TRAFFIC_CLASS - 1)
                        sbuf_printf(sbuf, ",");
        }

        sbuf_finish(sbuf);
        sbuf_delete(sbuf);

        /* Read in the new DSCP mapping values */
        ret = sysctl_handle_string(oidp, dscp_user_buf, sizeof(dscp_user_buf), req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        /* Don't allow setting changes in FW DCB mode */
        if (!hw->port_info->qos_cfg.is_sw_lldp) {
                device_printf(dev, "%s: DSCP mapping is not allowed in FW DCBX mode\n",
                    __func__);
                return (EINVAL);
        }

        /* Convert 8 values in a string to a table; this is similar to what
         * needs to be done for ETS settings, so this function can be re-used
         * for that purpose.
         */
        ret = ice_ets_str_to_tbl(dscp_user_buf, new_dscp_table_seg,
            ICE_MAX_TRAFFIC_CLASS - 1);
        if (ret) {
                device_printf(dev, "%s: Could not parse input DSCP2TC table: %s\n",
                    __func__, dscp_user_buf);
                return (ret);
        }

        memcpy(&local_dcbx_cfg->dscp_map[arg2 * 8], new_dscp_table_seg,
            sizeof(new_dscp_table_seg));

        local_dcbx_cfg->app_mode = ICE_DCBX_APPS_NON_WILLING;

        status = ice_set_dcb_cfg(pi);
        if (status) {
                device_printf(dev,
                    "%s: Failed to set DCB config; status %s, aq_err %s\n",
                    __func__, ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        ice_do_dcb_reconfig(sc, false);

        return (0);
}

/**
 * ice_handle_debug_dump_ioctl - Handle a debug dump ioctl request
 * @sc: the device private softc
 * @ifd: ifdrv ioctl request pointer
 */
int
ice_handle_debug_dump_ioctl(struct ice_softc *sc, struct ifdrv *ifd)
{
        size_t ifd_len = ifd->ifd_len;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        struct ice_debug_dump_cmd *ddc;
        int status;
        int err = 0;

        /* Returned arguments from the Admin Queue */
        u16 ret_buf_size = 0;
        u16 ret_next_cluster = 0;
        u16 ret_next_table = 0;
        u32 ret_next_index = 0;

        /*
         * ifioctl forwards SIOCxDRVSPEC to iflib without performing
         * a privilege check. In turn, iflib forwards the ioctl to the driver
         * without performing a privilege check. Perform one here to ensure
         * that non-privileged threads cannot access this interface.
         */
        err = priv_check(curthread, PRIV_DRIVER);
        if (err)
                return (err);

        if (ice_test_state(&sc->state, ICE_STATE_PREPARED_FOR_RESET)) {
                device_printf(dev,
                    "%s: Driver must rebuild data structures after a reset. Operation aborted.\n",
                    __func__);
                return (EBUSY);
        }

        if (ifd_len < sizeof(*ddc)) {
                device_printf(dev,
                    "%s: ifdrv length is too small. Got %zu, but expected %zu\n",
                    __func__, ifd_len, sizeof(*ddc));
                return (EINVAL);
        }

        if (ifd->ifd_data == NULL) {
                device_printf(dev, "%s: ifd data buffer not present.\n",
                     __func__);
                return (EINVAL);
        }

        ddc = (struct ice_debug_dump_cmd *)malloc(ifd_len, M_ICE, M_ZERO | M_NOWAIT);
        if (!ddc)
                return (ENOMEM);

        /* Copy the NVM access command and data in from user space */
        /* coverity[tainted_data_argument] */
        err = copyin(ifd->ifd_data, ddc, ifd_len);
        if (err) {
                device_printf(dev, "%s: Copying request from user space failed, err %s\n",
                              __func__, ice_err_str(err));
                goto out;
        }

        /* The data_size arg must be at least 1 for the AQ cmd to work */
        if (ddc->data_size == 0) {
                device_printf(dev,
                    "%s: data_size must be greater than 0\n", __func__);
                err = EINVAL;
                goto out;
        }
        /* ...and it can't be too long */
        if (ddc->data_size > (ifd_len - sizeof(*ddc))) {
                device_printf(dev,
                    "%s: data_size (%d) is larger than ifd_len space (%zu)?\n", __func__,
                    ddc->data_size, ifd_len - sizeof(*ddc));
                err = EINVAL;
                goto out;
        }

        /* Make sure any possible data buffer space is zeroed */
        memset(ddc->data, 0, ifd_len - sizeof(*ddc));

        status = ice_aq_get_internal_data(hw, ddc->cluster_id, ddc->table_id, ddc->offset,
            (u8 *)ddc->data, ddc->data_size, &ret_buf_size, 
            &ret_next_cluster, &ret_next_table, &ret_next_index, NULL);
        ice_debug(hw, ICE_DBG_DIAG, "%s: ret_buf_size %d, ret_next_table %d, ret_next_index %d\n",
            __func__, ret_buf_size, ret_next_table, ret_next_index);
        if (status) {
                device_printf(dev,
                    "%s: Get Internal Data AQ command failed, err %s aq_err %s\n",
                    __func__,
                    ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                goto aq_error;
        }

        ddc->table_id = ret_next_table;
        ddc->offset = ret_next_index;
        ddc->data_size = ret_buf_size;
        ddc->cluster_id = ret_next_cluster;

        /* Copy the possibly modified contents of the handled request out */
        err = copyout(ddc, ifd->ifd_data, ifd->ifd_len);
        if (err) {
                device_printf(dev, "%s: Copying response back to user space failed, err %s\n",
                              __func__, ice_err_str(err));
                goto out;
        }

aq_error:
        /* Convert private status to an error code for proper ioctl response */
        switch (status) {
        case 0:
                err = (0);
                break;
        case ICE_ERR_NO_MEMORY:
                err = (ENOMEM);
                break;
        case ICE_ERR_OUT_OF_RANGE:
                err = (ENOTTY);
                break;
        case ICE_ERR_AQ_ERROR:
                err = (EIO);
                break;
        case ICE_ERR_PARAM:
        default:
                err = (EINVAL);
                break;
        }

out:
        free(ddc, M_ICE);
        return (err);
}

/**
 * ice_sysctl_allow_no_fec_mod_in_auto - Change Auto FEC behavior
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * Allows user to let "No FEC" mode to be used in "Auto"
 * FEC mode during FEC negotiation. This is only supported
 * on newer firmware versions.
 */
static int
ice_sysctl_allow_no_fec_mod_in_auto(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        u8 user_flag;
        int ret;

        UNREFERENCED_PARAMETER(arg2);

        ret = priv_check(curthread, PRIV_DRIVER);
        if (ret)
                return (ret);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        user_flag = (u8)sc->allow_no_fec_mod_in_auto;

        ret = sysctl_handle_bool(oidp, &user_flag, 0, req);
        if ((ret) || (req->newptr == NULL))
                return (ret);

        if (!ice_fw_supports_fec_dis_auto(hw)) {
                log(LOG_INFO,
                    "%s: Enabling or disabling of auto configuration of modules that don't support FEC is unsupported by the current firmware\n",
                    device_get_nameunit(dev));
                return (ENODEV);
        }

        if (user_flag == (bool)sc->allow_no_fec_mod_in_auto)
                return (0);

        sc->allow_no_fec_mod_in_auto = (u8)user_flag;

        if (sc->allow_no_fec_mod_in_auto)
                log(LOG_INFO, "%s: Enabled auto configuration of No FEC modules\n",
                    device_get_nameunit(dev));
        else
                log(LOG_INFO,
                    "%s: Auto configuration of No FEC modules reset to NVM defaults\n",
                    device_get_nameunit(dev));

        return (0);
}

/**
 * ice_sysctl_temperature - Retrieve NIC temp via AQ command
 * @oidp: sysctl oid structure
 * @arg1: pointer to private data structure
 * @arg2: unused
 * @req: sysctl request pointer
 *
 * If ICE_DBG_DIAG is set in the debug.debug_mask sysctl, then this will print
 * temperature threshold information in the kernel message log, too.
 */
static int
ice_sysctl_temperature(SYSCTL_HANDLER_ARGS)
{
        struct ice_aqc_get_sensor_reading_resp resp;
        struct ice_softc *sc = (struct ice_softc *)arg1;
        struct ice_hw *hw = &sc->hw;
        device_t dev = sc->dev;
        int status;

        UNREFERENCED_PARAMETER(oidp);
        UNREFERENCED_PARAMETER(arg2);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        status = ice_aq_get_sensor_reading(hw, ICE_AQC_INT_TEMP_SENSOR,
            ICE_AQC_INT_TEMP_FORMAT, &resp, NULL);
        if (status) {
                device_printf(dev,
                    "Get Sensor Reading AQ call failed, err %s aq_err %s\n",
                    ice_status_str(status),
                    ice_aq_str(hw->adminq.sq_last_status));
                return (EIO);
        }

        ice_debug(hw, ICE_DBG_DIAG, "%s: Warning Temp Threshold: %d\n", __func__,
            resp.data.s0f0.temp_warning_threshold);
        ice_debug(hw, ICE_DBG_DIAG, "%s: Critical Temp Threshold: %d\n", __func__,
            resp.data.s0f0.temp_critical_threshold);
        ice_debug(hw, ICE_DBG_DIAG, "%s: Fatal Temp Threshold: %d\n", __func__,
            resp.data.s0f0.temp_fatal_threshold);

        return sysctl_handle_8(oidp, &resp.data.s0f0.temp, 0, req);
}

/**
 * ice_sysctl_create_mirror_interface - Create a new ifnet that monitors
 *     traffic from the main PF VSI
 */
static int
ice_sysctl_create_mirror_interface(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        device_t dev = sc->dev;
        int ret;

        UNREFERENCED_PARAMETER(arg2);

        ret = priv_check(curthread, PRIV_DRIVER);
        if (ret)
                return (ret);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        /* If the user hasn't written "1" to this sysctl yet: */
        if (!ice_test_state(&sc->state, ICE_STATE_DO_CREATE_MIRR_INTFC)) {
                /* Avoid output on the first set of reads to this sysctl in
                 * order to prevent a null byte from being written to the
                 * end result when called via sysctl(8).
                 */
                if (req->oldptr == NULL && req->newptr == NULL) {
                        ret = SYSCTL_OUT(req, 0, 0);
                        return (ret);
                }

                char input_buf[2] = "";
                ret = sysctl_handle_string(oidp, input_buf, sizeof(input_buf), req);
                if ((ret) || (req->newptr == NULL))
                        return (ret);

                /* If we get '1', then indicate we'll create the interface in
                 * the next sysctl read call.
                 */
                if (input_buf[0] == '1') {
                        if (sc->mirr_if) {
                                device_printf(dev,
                                    "Mirror interface %s already exists!\n",
                                    if_name(sc->mirr_if->ifp));
                                return (EEXIST);
                        }
                        ice_set_state(&sc->state, ICE_STATE_DO_CREATE_MIRR_INTFC);
                        return (0);
                }

                return (EINVAL);
        }

        /* --- "Do Create Mirror Interface" is set --- */

        /* Caller just wants the upper bound for size */
        if (req->oldptr == NULL && req->newptr == NULL) {
                ret = SYSCTL_OUT(req, 0, 128);
                return (ret);
        }

        device_printf(dev, "Creating new mirroring interface...\n");

        ret = ice_create_mirror_interface(sc);
        if (ret)
                return (ret);

        ice_clear_state(&sc->state, ICE_STATE_DO_CREATE_MIRR_INTFC);

        ret = sysctl_handle_string(oidp, __DECONST(char *, "Interface attached"), 0, req);
        return (ret);
}

/**
 * ice_sysctl_destroy_mirror_interface - Destroy network interface that monitors
 *     traffic from the main PF VSI
 */
static int
ice_sysctl_destroy_mirror_interface(SYSCTL_HANDLER_ARGS)
{
        struct ice_softc *sc = (struct ice_softc *)arg1;
        device_t dev = sc->dev;
        int ret;

        UNREFERENCED_PARAMETER(arg2);

        ret = priv_check(curthread, PRIV_DRIVER);
        if (ret)
                return (ret);

        if (ice_driver_is_detaching(sc))
                return (ESHUTDOWN);

        /* If the user hasn't written "1" to this sysctl yet: */
        if (!ice_test_state(&sc->state, ICE_STATE_DO_DESTROY_MIRR_INTFC)) {
                /* Avoid output on the first set of reads to this sysctl in
                 * order to prevent a null byte from being written to the
                 * end result when called via sysctl(8).
                 */
                if (req->oldptr == NULL && req->newptr == NULL) {
                        ret = SYSCTL_OUT(req, 0, 0);
                        return (ret);
                }

                char input_buf[2] = "";
                ret = sysctl_handle_string(oidp, input_buf, sizeof(input_buf), req);
                if ((ret) || (req->newptr == NULL))
                        return (ret);

                /* If we get '1', then indicate we'll create the interface in
                 * the next sysctl read call.
                 */
                if (input_buf[0] == '1') {
                        if (!sc->mirr_if) {
                                device_printf(dev,
                                    "No mirror interface exists!\n");
                                return (EINVAL);
                        }
                        ice_set_state(&sc->state, ICE_STATE_DO_DESTROY_MIRR_INTFC);
                        return (0);
                }

                return (EINVAL);
        }

        /* --- "Do Destroy Mirror Interface" is set --- */

        /* Caller just wants the upper bound for size */
        if (req->oldptr == NULL && req->newptr == NULL) {
                ret = SYSCTL_OUT(req, 0, 128);
                return (ret);
        }

        device_printf(dev, "Destroying mirroring interface...\n");

        ice_destroy_mirror_interface(sc);

        ice_clear_state(&sc->state, ICE_STATE_DO_DESTROY_MIRR_INTFC);

        ret = sysctl_handle_string(oidp, __DECONST(char *, "Interface destroyed"), 0, req);
        return (ret);
}