/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2008, 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * hermon_ci.c
 *    Hermon Channel Interface (CI) Routines
 *
 *    Implements all the routines necessary to interface with the IBTF.
 *    Pointers to all of these functions are passed to the IBTF at attach()
 *    time in the ibc_operations_t structure.  These functions include all
 *    of the necessary routines to implement the required InfiniBand "verbs"
 *    and additional IBTF-specific interfaces.
 */

#include <sys/types.h>
#include <sys/conf.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>

#include <sys/ib/adapters/hermon/hermon.h>

extern uint32_t hermon_kernel_data_ro;
extern uint32_t hermon_user_data_ro;

/* HCA and port related operations */
static ibt_status_t hermon_ci_query_hca_ports(ibc_hca_hdl_t, uint8_t,
    ibt_hca_portinfo_t *);
static ibt_status_t hermon_ci_modify_ports(ibc_hca_hdl_t, uint8_t,
    ibt_port_modify_flags_t, uint8_t);
static ibt_status_t hermon_ci_modify_system_image(ibc_hca_hdl_t, ib_guid_t);

/* Protection Domains */
static ibt_status_t hermon_ci_alloc_pd(ibc_hca_hdl_t, ibt_pd_flags_t,
    ibc_pd_hdl_t *);
static ibt_status_t hermon_ci_free_pd(ibc_hca_hdl_t, ibc_pd_hdl_t);

/* Reliable Datagram Domains */
static ibt_status_t hermon_ci_alloc_rdd(ibc_hca_hdl_t, ibc_rdd_flags_t,
    ibc_rdd_hdl_t *);
static ibt_status_t hermon_ci_free_rdd(ibc_hca_hdl_t, ibc_rdd_hdl_t);

/* Address Handles */
static ibt_status_t hermon_ci_alloc_ah(ibc_hca_hdl_t, ibt_ah_flags_t,
    ibc_pd_hdl_t, ibt_adds_vect_t *, ibc_ah_hdl_t *);
static ibt_status_t hermon_ci_free_ah(ibc_hca_hdl_t, ibc_ah_hdl_t);
static ibt_status_t hermon_ci_query_ah(ibc_hca_hdl_t, ibc_ah_hdl_t,
    ibc_pd_hdl_t *, ibt_adds_vect_t *);
static ibt_status_t hermon_ci_modify_ah(ibc_hca_hdl_t, ibc_ah_hdl_t,
    ibt_adds_vect_t *);

/* Queue Pairs */
static ibt_status_t hermon_ci_alloc_qp(ibc_hca_hdl_t, ibtl_qp_hdl_t,
    ibt_qp_type_t, ibt_qp_alloc_attr_t *, ibt_chan_sizes_t *, ib_qpn_t *,
    ibc_qp_hdl_t *);
static ibt_status_t hermon_ci_alloc_special_qp(ibc_hca_hdl_t, uint8_t,
    ibtl_qp_hdl_t, ibt_sqp_type_t, ibt_qp_alloc_attr_t *,
    ibt_chan_sizes_t *, ibc_qp_hdl_t *);
static ibt_status_t hermon_ci_alloc_qp_range(ibc_hca_hdl_t, uint_t,
    ibtl_qp_hdl_t *, ibt_qp_type_t, ibt_qp_alloc_attr_t *, ibt_chan_sizes_t *,
    ibc_cq_hdl_t *, ibc_cq_hdl_t *, ib_qpn_t *, ibc_qp_hdl_t *);
static ibt_status_t hermon_ci_free_qp(ibc_hca_hdl_t, ibc_qp_hdl_t,
    ibc_free_qp_flags_t, ibc_qpn_hdl_t *);
static ibt_status_t hermon_ci_release_qpn(ibc_hca_hdl_t, ibc_qpn_hdl_t);
static ibt_status_t hermon_ci_query_qp(ibc_hca_hdl_t, ibc_qp_hdl_t,
    ibt_qp_query_attr_t *);
static ibt_status_t hermon_ci_modify_qp(ibc_hca_hdl_t, ibc_qp_hdl_t,
    ibt_cep_modify_flags_t, ibt_qp_info_t *, ibt_queue_sizes_t *);

/* Completion Queues */
static ibt_status_t hermon_ci_alloc_cq(ibc_hca_hdl_t, ibt_cq_hdl_t,
    ibt_cq_attr_t *, ibc_cq_hdl_t *, uint_t *);
static ibt_status_t hermon_ci_free_cq(ibc_hca_hdl_t, ibc_cq_hdl_t);
static ibt_status_t hermon_ci_query_cq(ibc_hca_hdl_t, ibc_cq_hdl_t,
    uint_t *, uint_t *, uint_t *, ibt_cq_handler_id_t *);
static ibt_status_t hermon_ci_resize_cq(ibc_hca_hdl_t, ibc_cq_hdl_t,
    uint_t, uint_t *);
static ibt_status_t hermon_ci_modify_cq(ibc_hca_hdl_t, ibc_cq_hdl_t,
    uint_t, uint_t, ibt_cq_handler_id_t);
static ibt_status_t hermon_ci_alloc_cq_sched(ibc_hca_hdl_t,
    ibt_cq_sched_attr_t *, ibc_sched_hdl_t *);
static ibt_status_t hermon_ci_free_cq_sched(ibc_hca_hdl_t, ibc_sched_hdl_t);
static ibt_status_t hermon_ci_query_cq_handler_id(ibc_hca_hdl_t,
    ibt_cq_handler_id_t, ibt_cq_handler_attr_t *);

/* EE Contexts */
static ibt_status_t hermon_ci_alloc_eec(ibc_hca_hdl_t, ibc_eec_flags_t,
    ibt_eec_hdl_t, ibc_rdd_hdl_t, ibc_eec_hdl_t *);
static ibt_status_t hermon_ci_free_eec(ibc_hca_hdl_t, ibc_eec_hdl_t);
static ibt_status_t hermon_ci_query_eec(ibc_hca_hdl_t, ibc_eec_hdl_t,
    ibt_eec_query_attr_t *);
static ibt_status_t hermon_ci_modify_eec(ibc_hca_hdl_t, ibc_eec_hdl_t,
    ibt_cep_modify_flags_t, ibt_eec_info_t *);

/* Memory Registration */
static ibt_status_t hermon_ci_register_mr(ibc_hca_hdl_t, ibc_pd_hdl_t,
    ibt_mr_attr_t *, void *, ibc_mr_hdl_t *, ibt_mr_desc_t *);
static ibt_status_t hermon_ci_register_buf(ibc_hca_hdl_t, ibc_pd_hdl_t,
    ibt_smr_attr_t *, struct buf *, void *, ibt_mr_hdl_t *, ibt_mr_desc_t *);
static ibt_status_t hermon_ci_register_shared_mr(ibc_hca_hdl_t,
    ibc_mr_hdl_t, ibc_pd_hdl_t, ibt_smr_attr_t *, void *,
    ibc_mr_hdl_t *, ibt_mr_desc_t *);
static ibt_status_t hermon_ci_deregister_mr(ibc_hca_hdl_t, ibc_mr_hdl_t);
static ibt_status_t hermon_ci_query_mr(ibc_hca_hdl_t, ibc_mr_hdl_t,
    ibt_mr_query_attr_t *);
static ibt_status_t hermon_ci_reregister_mr(ibc_hca_hdl_t, ibc_mr_hdl_t,
    ibc_pd_hdl_t, ibt_mr_attr_t *, void *, ibc_mr_hdl_t *,
    ibt_mr_desc_t *);
static ibt_status_t hermon_ci_reregister_buf(ibc_hca_hdl_t, ibc_mr_hdl_t,
    ibc_pd_hdl_t, ibt_smr_attr_t *, struct buf *, void *, ibc_mr_hdl_t *,
    ibt_mr_desc_t *);
static ibt_status_t hermon_ci_sync_mr(ibc_hca_hdl_t, ibt_mr_sync_t *, size_t);
static ibt_status_t hermon_ci_register_dma_mr(ibc_hca_hdl_t, ibc_pd_hdl_t,
    ibt_dmr_attr_t *, void *, ibc_mr_hdl_t *, ibt_mr_desc_t *);

/* Memory Windows */
static ibt_status_t hermon_ci_alloc_mw(ibc_hca_hdl_t, ibc_pd_hdl_t,
    ibt_mw_flags_t, ibc_mw_hdl_t *, ibt_rkey_t *);
static ibt_status_t hermon_ci_free_mw(ibc_hca_hdl_t, ibc_mw_hdl_t);
static ibt_status_t hermon_ci_query_mw(ibc_hca_hdl_t, ibc_mw_hdl_t,
    ibt_mw_query_attr_t *);

/* Multicast Groups */
static ibt_status_t hermon_ci_attach_mcg(ibc_hca_hdl_t, ibc_qp_hdl_t,
    ib_gid_t, ib_lid_t);
static ibt_status_t hermon_ci_detach_mcg(ibc_hca_hdl_t, ibc_qp_hdl_t,
    ib_gid_t, ib_lid_t);

/* Work Request and Completion Processing */
static ibt_status_t hermon_ci_post_send(ibc_hca_hdl_t, ibc_qp_hdl_t,
    ibt_send_wr_t *, uint_t, uint_t *);
static ibt_status_t hermon_ci_post_recv(ibc_hca_hdl_t, ibc_qp_hdl_t,
    ibt_recv_wr_t *, uint_t, uint_t *);
static ibt_status_t hermon_ci_poll_cq(ibc_hca_hdl_t, ibc_cq_hdl_t,
    ibt_wc_t *, uint_t, uint_t *);
static ibt_status_t hermon_ci_notify_cq(ibc_hca_hdl_t, ibc_cq_hdl_t,
    ibt_cq_notify_flags_t);

/* CI Object Private Data */
static ibt_status_t hermon_ci_ci_data_in(ibc_hca_hdl_t, ibt_ci_data_flags_t,
    ibt_object_type_t, void *, void *, size_t);

/* CI Object Private Data */
static ibt_status_t hermon_ci_ci_data_out(ibc_hca_hdl_t, ibt_ci_data_flags_t,
    ibt_object_type_t, void *, void *, size_t);

/* Shared Receive Queues */
static ibt_status_t hermon_ci_alloc_srq(ibc_hca_hdl_t, ibt_srq_flags_t,
    ibt_srq_hdl_t, ibc_pd_hdl_t, ibt_srq_sizes_t *, ibc_srq_hdl_t *,
    ibt_srq_sizes_t *);
static ibt_status_t hermon_ci_free_srq(ibc_hca_hdl_t, ibc_srq_hdl_t);
static ibt_status_t hermon_ci_query_srq(ibc_hca_hdl_t, ibc_srq_hdl_t,
    ibc_pd_hdl_t *, ibt_srq_sizes_t *, uint_t *);
static ibt_status_t hermon_ci_modify_srq(ibc_hca_hdl_t, ibc_srq_hdl_t,
    ibt_srq_modify_flags_t, uint_t, uint_t, uint_t *);
static ibt_status_t hermon_ci_post_srq(ibc_hca_hdl_t, ibc_srq_hdl_t,
    ibt_recv_wr_t *, uint_t, uint_t *);

/* Address translation */
static ibt_status_t hermon_ci_map_mem_area(ibc_hca_hdl_t, ibt_va_attr_t *,
    void *, uint_t, ibt_reg_req_t *, ibc_ma_hdl_t *);
static ibt_status_t hermon_ci_unmap_mem_area(ibc_hca_hdl_t, ibc_ma_hdl_t);
static ibt_status_t hermon_ci_map_mem_iov(ibc_hca_hdl_t, ibt_iov_attr_t *,
    ibt_all_wr_t *, ibc_mi_hdl_t *);
static ibt_status_t hermon_ci_unmap_mem_iov(ibc_hca_hdl_t, ibc_mi_hdl_t);

/* Allocate L_Key */
static ibt_status_t hermon_ci_alloc_lkey(ibc_hca_hdl_t, ibc_pd_hdl_t,
    ibt_lkey_flags_t, uint_t, ibc_mr_hdl_t *, ibt_pmr_desc_t *);

/* Physical Register Memory Region */
static ibt_status_t hermon_ci_register_physical_mr(ibc_hca_hdl_t, ibc_pd_hdl_t,
    ibt_pmr_attr_t *, void *, ibc_mr_hdl_t *, ibt_pmr_desc_t *);
static ibt_status_t hermon_ci_reregister_physical_mr(ibc_hca_hdl_t,
    ibc_mr_hdl_t, ibc_pd_hdl_t, ibt_pmr_attr_t *, void *, ibc_mr_hdl_t *,
    ibt_pmr_desc_t *);

/* Mellanox FMR */
static ibt_status_t hermon_ci_create_fmr_pool(ibc_hca_hdl_t hca,
    ibc_pd_hdl_t pd, ibt_fmr_pool_attr_t *fmr_params,
    ibc_fmr_pool_hdl_t *fmr_pool);
static ibt_status_t hermon_ci_destroy_fmr_pool(ibc_hca_hdl_t hca,
    ibc_fmr_pool_hdl_t fmr_pool);
static ibt_status_t hermon_ci_flush_fmr_pool(ibc_hca_hdl_t hca,
    ibc_fmr_pool_hdl_t fmr_pool);
static ibt_status_t hermon_ci_register_physical_fmr(ibc_hca_hdl_t hca,
    ibc_fmr_pool_hdl_t fmr_pool, ibt_pmr_attr_t *mem_pattr,
    void *ibtl_reserved, ibc_mr_hdl_t *mr_hdl_p, ibt_pmr_desc_t *mem_desc_p);
static ibt_status_t hermon_ci_deregister_fmr(ibc_hca_hdl_t hca,
    ibc_mr_hdl_t mr);

/* Memory Allocation/Deallocation */
static ibt_status_t hermon_ci_alloc_io_mem(ibc_hca_hdl_t hca, size_t size,
    ibt_mr_flags_t mr_flag, caddr_t *kaddrp,
    ibc_mem_alloc_hdl_t *mem_alloc_hdl_p);
static ibt_status_t hermon_ci_free_io_mem(ibc_hca_hdl_t hca,
    ibc_mem_alloc_hdl_t mem_alloc_hdl);
static ibt_status_t hermon_ci_not_supported();

/*
 * This ibc_operations_t structure includes pointers to all the entry points
 * provided by the Hermon driver.  This structure is passed to the IBTF at
 * driver attach time, using the ibc_attach() call.
 */
ibc_operations_t hermon_ibc_ops = {
        /* HCA and port related operations */
        hermon_ci_query_hca_ports,
        hermon_ci_modify_ports,
        hermon_ci_modify_system_image,

        /* Protection Domains */
        hermon_ci_alloc_pd,
        hermon_ci_free_pd,

        /* Reliable Datagram Domains */
        hermon_ci_alloc_rdd,
        hermon_ci_free_rdd,

        /* Address Handles */
        hermon_ci_alloc_ah,
        hermon_ci_free_ah,
        hermon_ci_query_ah,
        hermon_ci_modify_ah,

        /* Queue Pairs */
        hermon_ci_alloc_qp,
        hermon_ci_alloc_special_qp,
        hermon_ci_alloc_qp_range,
        hermon_ci_free_qp,
        hermon_ci_release_qpn,
        hermon_ci_query_qp,
        hermon_ci_modify_qp,

        /* Completion Queues */
        hermon_ci_alloc_cq,
        hermon_ci_free_cq,
        hermon_ci_query_cq,
        hermon_ci_resize_cq,
        hermon_ci_modify_cq,
        hermon_ci_alloc_cq_sched,
        hermon_ci_free_cq_sched,
        hermon_ci_query_cq_handler_id,

        /* EE Contexts */
        hermon_ci_alloc_eec,
        hermon_ci_free_eec,
        hermon_ci_query_eec,
        hermon_ci_modify_eec,

        /* Memory Registration */
        hermon_ci_register_mr,
        hermon_ci_register_buf,
        hermon_ci_register_shared_mr,
        hermon_ci_deregister_mr,
        hermon_ci_query_mr,
        hermon_ci_reregister_mr,
        hermon_ci_reregister_buf,
        hermon_ci_sync_mr,

        /* Memory Windows */
        hermon_ci_alloc_mw,
        hermon_ci_free_mw,
        hermon_ci_query_mw,

        /* Multicast Groups */
        hermon_ci_attach_mcg,
        hermon_ci_detach_mcg,

        /* Work Request and Completion Processing */
        hermon_ci_post_send,
        hermon_ci_post_recv,
        hermon_ci_poll_cq,
        hermon_ci_notify_cq,

        /* CI Object Mapping Data */
        hermon_ci_ci_data_in,
        hermon_ci_ci_data_out,

        /* Shared Receive Queue */
        hermon_ci_alloc_srq,
        hermon_ci_free_srq,
        hermon_ci_query_srq,
        hermon_ci_modify_srq,
        hermon_ci_post_srq,

        /* Address translation */
        hermon_ci_map_mem_area,
        hermon_ci_unmap_mem_area,
        hermon_ci_map_mem_iov,
        hermon_ci_unmap_mem_iov,

        /* Allocate L_key */
        hermon_ci_alloc_lkey,

        /* Physical Register Memory Region */
        hermon_ci_register_physical_mr,
        hermon_ci_reregister_physical_mr,

        /* Mellanox FMR */
        hermon_ci_create_fmr_pool,
        hermon_ci_destroy_fmr_pool,
        hermon_ci_flush_fmr_pool,
        hermon_ci_register_physical_fmr,
        hermon_ci_deregister_fmr,

        /* Memory allocation */
        hermon_ci_alloc_io_mem,
        hermon_ci_free_io_mem,

        /* XRC not yet supported */
        hermon_ci_not_supported,        /* ibc_alloc_xrc_domain */
        hermon_ci_not_supported,        /* ibc_free_xrc_domain */
        hermon_ci_not_supported,        /* ibc_alloc_xrc_srq */
        hermon_ci_not_supported,        /* ibc_free_xrc_srq */
        hermon_ci_not_supported,        /* ibc_query_xrc_srq */
        hermon_ci_not_supported,        /* ibc_modify_xrc_srq */
        hermon_ci_not_supported,        /* ibc_alloc_xrc_tgt_qp */
        hermon_ci_not_supported,        /* ibc_free_xrc_tgt_qp */
        hermon_ci_not_supported,        /* ibc_query_xrc_tgt_qp */
        hermon_ci_not_supported,        /* ibc_modify_xrc_tgt_qp */

        /* Memory Region (physical) */
        hermon_ci_register_dma_mr,

        /* Next enhancements */
        hermon_ci_not_supported,        /* ibc_enhancement1 */
        hermon_ci_not_supported,        /* ibc_enhancement2 */
        hermon_ci_not_supported,        /* ibc_enhancement3 */
        hermon_ci_not_supported,        /* ibc_enhancement4 */
};

/*
 * Not yet implemented OPS
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_not_supported()
{
        return (IBT_NOT_SUPPORTED);
}


/*
 * hermon_ci_query_hca_ports()
 *    Returns HCA port attributes for either one or all of the HCA's ports.
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_query_hca_ports(ibc_hca_hdl_t hca, uint8_t query_port,
    ibt_hca_portinfo_t *info_p)
{
        hermon_state_t  *state;
        uint_t          start, end, port;
        int             status, indx;

        /* Grab the Hermon softstate pointer */
        state = (hermon_state_t *)hca;

        /*
         * If the specified port is zero, then we are supposed to query all
         * ports.  Otherwise, we query only the port number specified.
         * Setup the start and end port numbers as appropriate for the loop
         * below.  Note:  The first Hermon port is port number one (1).
         */
        if (query_port == 0) {
                start = 1;
                end = start + (state->hs_cfg_profile->cp_num_ports - 1);
        } else {
                end = start = query_port;
        }

        /* Query the port(s) */
        for (port = start, indx = 0; port <= end; port++, indx++) {
                status = hermon_port_query(state, port, &info_p[indx]);
                if (status != DDI_SUCCESS) {
                        return (status);
                }
        }
        return (IBT_SUCCESS);
}


/*
 * hermon_ci_modify_ports()
 *    Modify HCA port attributes
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_modify_ports(ibc_hca_hdl_t hca, uint8_t port,
    ibt_port_modify_flags_t flags, uint8_t init_type)
{
        hermon_state_t  *state;
        int             status;

        /* Grab the Hermon softstate pointer */
        state = (hermon_state_t *)hca;

        /* Modify the port(s) */
        status = hermon_port_modify(state, port, flags, init_type);
        return (status);
}

/*
 * hermon_ci_modify_system_image()
 *    Modify the System Image GUID
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_modify_system_image(ibc_hca_hdl_t hca, ib_guid_t sys_guid)
{
        /*
         * This is an unsupported interface for the Hermon driver.  This
         * interface is necessary to support modification of the System
         * Image GUID.  Hermon is only capable of modifying this parameter
         * once (during driver initialization).
         */
        return (IBT_NOT_SUPPORTED);
}

/*
 * hermon_ci_alloc_pd()
 *    Allocate a Protection Domain
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_pd(ibc_hca_hdl_t hca, ibt_pd_flags_t flags, ibc_pd_hdl_t *pd_p)
{
        hermon_state_t  *state;
        hermon_pdhdl_t  pdhdl;
        int             status;

        ASSERT(pd_p != NULL);

        /* Grab the Hermon softstate pointer */
        state = (hermon_state_t *)hca;

        /* Allocate the PD */
        status = hermon_pd_alloc(state, &pdhdl, HERMON_NOSLEEP);
        if (status != DDI_SUCCESS) {
                return (status);
        }

        /* Return the Hermon PD handle */
        *pd_p = (ibc_pd_hdl_t)pdhdl;

        return (IBT_SUCCESS);
}


/*
 * hermon_ci_free_pd()
 *    Free a Protection Domain
 *    Context: Can be called only from user or kernel context
 */
static ibt_status_t
hermon_ci_free_pd(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd)
{
        hermon_state_t          *state;
        hermon_pdhdl_t          pdhdl;
        int                     status;

        /* Grab the Hermon softstate pointer and PD handle */
        state = (hermon_state_t *)hca;
        pdhdl = (hermon_pdhdl_t)pd;

        /* Free the PD */
        status = hermon_pd_free(state, &pdhdl);
        return (status);
}


/*
 * hermon_ci_alloc_rdd()
 *    Allocate a Reliable Datagram Domain
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_rdd(ibc_hca_hdl_t hca, ibc_rdd_flags_t flags,
    ibc_rdd_hdl_t *rdd_p)
{
        /*
         * This is an unsupported interface for the Hermon driver.  This
         * interface is necessary to support Reliable Datagram (RD)
         * operations.  Hermon does not support RD.
         */
        return (IBT_NOT_SUPPORTED);
}


/*
 * hermon_free_rdd()
 *    Free a Reliable Datagram Domain
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_free_rdd(ibc_hca_hdl_t hca, ibc_rdd_hdl_t rdd)
{
        /*
         * This is an unsupported interface for the Hermon driver.  This
         * interface is necessary to support Reliable Datagram (RD)
         * operations.  Hermon does not support RD.
         */
        return (IBT_NOT_SUPPORTED);
}


/*
 * hermon_ci_alloc_ah()
 *    Allocate an Address Handle
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_ah(ibc_hca_hdl_t hca, ibt_ah_flags_t flags, ibc_pd_hdl_t pd,
    ibt_adds_vect_t *attr_p, ibc_ah_hdl_t *ah_p)
{
        hermon_state_t  *state;
        hermon_ahhdl_t  ahhdl;
        hermon_pdhdl_t  pdhdl;
        int             status;

        /* Grab the Hermon softstate pointer and PD handle */
        state = (hermon_state_t *)hca;
        pdhdl = (hermon_pdhdl_t)pd;

        /* Allocate the AH */
        status = hermon_ah_alloc(state, pdhdl, attr_p, &ahhdl, HERMON_NOSLEEP);
        if (status != DDI_SUCCESS) {
                return (status);
        }

        /* Return the Hermon AH handle */
        *ah_p = (ibc_ah_hdl_t)ahhdl;

        return (IBT_SUCCESS);
}


/*
 * hermon_ci_free_ah()
 *    Free an Address Handle
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_free_ah(ibc_hca_hdl_t hca, ibc_ah_hdl_t ah)
{
        hermon_state_t  *state;
        hermon_ahhdl_t  ahhdl;
        int             status;

        /* Grab the Hermon softstate pointer and AH handle */
        state = (hermon_state_t *)hca;
        ahhdl = (hermon_ahhdl_t)ah;

        /* Free the AH */
        status = hermon_ah_free(state, &ahhdl, HERMON_NOSLEEP);

        return (status);
}


/*
 * hermon_ci_query_ah()
 *    Return the Address Vector information for a specified Address Handle
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_query_ah(ibc_hca_hdl_t hca, ibc_ah_hdl_t ah, ibc_pd_hdl_t *pd_p,
    ibt_adds_vect_t *attr_p)
{
        hermon_state_t  *state;
        hermon_ahhdl_t  ahhdl;
        hermon_pdhdl_t  pdhdl;
        int             status;

        /* Grab the Hermon softstate pointer and AH handle */
        state = (hermon_state_t *)hca;
        ahhdl = (hermon_ahhdl_t)ah;

        /* Query the AH */
        status = hermon_ah_query(state, ahhdl, &pdhdl, attr_p);
        if (status != DDI_SUCCESS) {
                return (status);
        }

        /* Return the Hermon PD handle */
        *pd_p = (ibc_pd_hdl_t)pdhdl;

        return (IBT_SUCCESS);
}


/*
 * hermon_ci_modify_ah()
 *    Modify the Address Vector information of a specified Address Handle
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_modify_ah(ibc_hca_hdl_t hca, ibc_ah_hdl_t ah, ibt_adds_vect_t *attr_p)
{
        hermon_state_t  *state;
        hermon_ahhdl_t  ahhdl;
        int             status;

        /* Grab the Hermon softstate pointer and AH handle */
        state = (hermon_state_t *)hca;
        ahhdl = (hermon_ahhdl_t)ah;

        /* Modify the AH */
        status = hermon_ah_modify(state, ahhdl, attr_p);

        return (status);
}


/*
 * hermon_ci_alloc_qp()
 *    Allocate a Queue Pair
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_alloc_qp(ibc_hca_hdl_t hca, ibtl_qp_hdl_t ibt_qphdl,
    ibt_qp_type_t type, ibt_qp_alloc_attr_t *attr_p,
    ibt_chan_sizes_t *queue_sizes_p, ib_qpn_t *qpn, ibc_qp_hdl_t *qp_p)
{
        hermon_state_t          *state;
        hermon_qp_info_t        qpinfo;
        int                     status;

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p))
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*queue_sizes_p))

        /* Grab the Hermon softstate pointer */
        state = (hermon_state_t *)hca;

        /* Allocate the QP */
        qpinfo.qpi_attrp        = attr_p;
        qpinfo.qpi_type         = type;
        qpinfo.qpi_ibt_qphdl    = ibt_qphdl;
        qpinfo.qpi_queueszp     = queue_sizes_p;
        qpinfo.qpi_qpn          = qpn;
        status = hermon_qp_alloc(state, &qpinfo, HERMON_NOSLEEP);
        if (status != DDI_SUCCESS) {
                return (status);
        }

        /* Return the Hermon QP handle */
        *qp_p = (ibc_qp_hdl_t)qpinfo.qpi_qphdl;

        return (IBT_SUCCESS);
}


/*
 * hermon_ci_alloc_special_qp()
 *    Allocate a Special Queue Pair
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_alloc_special_qp(ibc_hca_hdl_t hca, uint8_t port,
    ibtl_qp_hdl_t ibt_qphdl, ibt_sqp_type_t type,
    ibt_qp_alloc_attr_t *attr_p, ibt_chan_sizes_t *queue_sizes_p,
    ibc_qp_hdl_t *qp_p)
{
        hermon_state_t          *state;
        hermon_qp_info_t        qpinfo;
        int                     status;

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p))
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*queue_sizes_p))

        /* Grab the Hermon softstate pointer */
        state = (hermon_state_t *)hca;

        /* Allocate the Special QP */
        qpinfo.qpi_attrp        = attr_p;
        qpinfo.qpi_type         = type;
        qpinfo.qpi_port         = port;
        qpinfo.qpi_ibt_qphdl    = ibt_qphdl;
        qpinfo.qpi_queueszp     = queue_sizes_p;
        status = hermon_special_qp_alloc(state, &qpinfo, HERMON_NOSLEEP);
        if (status != DDI_SUCCESS) {
                return (status);
        }
        /* Return the Hermon QP handle */
        *qp_p = (ibc_qp_hdl_t)qpinfo.qpi_qphdl;

        return (IBT_SUCCESS);
}

/*
 * hermon_ci_alloc_qp_range()
 *    Free a Queue Pair
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_qp_range(ibc_hca_hdl_t hca, uint_t log2,
    ibtl_qp_hdl_t *ibtl_qp, ibt_qp_type_t type,
    ibt_qp_alloc_attr_t *attr_p, ibt_chan_sizes_t *queue_sizes_p,
    ibc_cq_hdl_t *send_cq, ibc_cq_hdl_t *recv_cq,
    ib_qpn_t *qpn, ibc_qp_hdl_t *qp_p)
{
        hermon_state_t          *state;
        hermon_qp_info_t        qpinfo;
        int                     status;

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attr_p))
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*queue_sizes_p))

        /* Grab the Hermon softstate pointer */
        state = (hermon_state_t *)hca;

        /* Allocate the QP */
        qpinfo.qpi_attrp        = attr_p;
        qpinfo.qpi_type         = type;
        qpinfo.qpi_queueszp     = queue_sizes_p;
        qpinfo.qpi_qpn          = qpn;
        status = hermon_qp_alloc_range(state, log2, &qpinfo, ibtl_qp,
            send_cq, recv_cq, (hermon_qphdl_t *)qp_p, HERMON_NOSLEEP);
        return (status);
}

/*
 * hermon_ci_free_qp()
 *    Free a Queue Pair
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_free_qp(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp,
    ibc_free_qp_flags_t free_qp_flags, ibc_qpn_hdl_t *qpnh_p)
{
        hermon_state_t  *state;
        hermon_qphdl_t  qphdl;
        int             status;

        /* Grab the Hermon softstate pointer and QP handle */
        state = (hermon_state_t *)hca;
        qphdl = (hermon_qphdl_t)qp;

        /* Free the QP */
        status = hermon_qp_free(state, &qphdl, free_qp_flags, qpnh_p,
            HERMON_NOSLEEP);

        return (status);
}


/*
 * hermon_ci_release_qpn()
 *    Release a Queue Pair Number (QPN)
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_release_qpn(ibc_hca_hdl_t hca, ibc_qpn_hdl_t qpnh)
{
        hermon_state_t          *state;
        hermon_qpn_entry_t      *entry;

        /* Grab the Hermon softstate pointer and QP handle */
        state = (hermon_state_t *)hca;
        entry = (hermon_qpn_entry_t *)qpnh;

        /* Release the QP number */
        hermon_qp_release_qpn(state, entry, HERMON_QPN_RELEASE);

        return (IBT_SUCCESS);
}


/*
 * hermon_ci_query_qp()
 *    Query a Queue Pair
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_query_qp(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp,
    ibt_qp_query_attr_t *attr_p)
{
        hermon_state_t  *state;
        hermon_qphdl_t  qphdl;
        int             status;

        /* Grab the Hermon softstate pointer and QP handle */
        state = (hermon_state_t *)hca;
        qphdl = (hermon_qphdl_t)qp;

        /* Query the QP */
        status = hermon_qp_query(state, qphdl, attr_p);
        return (status);
}


/*
 * hermon_ci_modify_qp()
 *    Modify a Queue Pair
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_modify_qp(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp,
    ibt_cep_modify_flags_t flags, ibt_qp_info_t *info_p,
    ibt_queue_sizes_t *actual_sz)
{
        hermon_state_t  *state;
        hermon_qphdl_t  qphdl;
        int             status;

        /* Grab the Hermon softstate pointer and QP handle */
        state = (hermon_state_t *)hca;
        qphdl = (hermon_qphdl_t)qp;

        /* Modify the QP */
        status = hermon_qp_modify(state, qphdl, flags, info_p, actual_sz);
        return (status);
}


/*
 * hermon_ci_alloc_cq()
 *    Allocate a Completion Queue
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_cq(ibc_hca_hdl_t hca, ibt_cq_hdl_t ibt_cqhdl,
    ibt_cq_attr_t *attr_p, ibc_cq_hdl_t *cq_p, uint_t *actual_size)
{
        hermon_state_t  *state;
        hermon_cqhdl_t  cqhdl;
        int             status;

        state = (hermon_state_t *)hca;

        /* Allocate the CQ */
        status = hermon_cq_alloc(state, ibt_cqhdl, attr_p, actual_size,
            &cqhdl, HERMON_NOSLEEP);
        if (status != DDI_SUCCESS) {
                return (status);
        }

        /* Return the Hermon CQ handle */
        *cq_p = (ibc_cq_hdl_t)cqhdl;

        return (IBT_SUCCESS);
}


/*
 * hermon_ci_free_cq()
 *    Free a Completion Queue
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_free_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq)
{
        hermon_state_t  *state;
        hermon_cqhdl_t  cqhdl;
        int             status;

        /* Grab the Hermon softstate pointer and CQ handle */
        state = (hermon_state_t *)hca;
        cqhdl = (hermon_cqhdl_t)cq;


        /* Free the CQ */
        status = hermon_cq_free(state, &cqhdl, HERMON_NOSLEEP);
        return (status);
}


/*
 * hermon_ci_query_cq()
 *    Return the size of a Completion Queue
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_query_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq, uint_t *entries_p,
    uint_t *count_p, uint_t *usec_p, ibt_cq_handler_id_t *hid_p)
{
        hermon_state_t  *state;
        hermon_cqhdl_t  cqhdl;

        /* Grab the CQ handle */
        state = (hermon_state_t *)hca;
        cqhdl = (hermon_cqhdl_t)cq;

        /* Query the current CQ size */
        *entries_p = cqhdl->cq_bufsz;
        *count_p = cqhdl->cq_intmod_count;
        *usec_p = cqhdl->cq_intmod_usec;
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*cqhdl))
        *hid_p = HERMON_EQNUM_TO_HID(state, cqhdl->cq_eqnum);

        return (IBT_SUCCESS);
}


/*
 * hermon_ci_resize_cq()
 *    Change the size of a Completion Queue
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_resize_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq, uint_t size,
    uint_t *actual_size)
{
        hermon_state_t          *state;
        hermon_cqhdl_t          cqhdl;
        int                     status;

        /* Grab the Hermon softstate pointer and CQ handle */
        state = (hermon_state_t *)hca;
        cqhdl = (hermon_cqhdl_t)cq;

        /* Resize the CQ */
        status = hermon_cq_resize(state, cqhdl, size, actual_size,
            HERMON_NOSLEEP);
        if (status != DDI_SUCCESS) {
                return (status);
        }
        return (IBT_SUCCESS);
}

/*
 * hermon_ci_modify_cq()
 *    Change the interrupt moderation values of a Completion Queue
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_modify_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq, uint_t count,
    uint_t usec, ibt_cq_handler_id_t hid)
{
        hermon_state_t          *state;
        hermon_cqhdl_t          cqhdl;
        int                     status;

        /* Grab the Hermon softstate pointer and CQ handle */
        state = (hermon_state_t *)hca;
        cqhdl = (hermon_cqhdl_t)cq;

        /* Resize the CQ */
        status = hermon_cq_modify(state, cqhdl, count, usec, hid,
            HERMON_NOSLEEP);
        return (status);
}


/*
 * hermon_ci_alloc_cq_sched()
 *    Reserve a CQ scheduling class resource
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_cq_sched(ibc_hca_hdl_t hca, ibt_cq_sched_attr_t *attr,
    ibc_sched_hdl_t *sched_hdl_p)
{
        int     status;

        status = hermon_cq_sched_alloc((hermon_state_t *)hca, attr,
            (hermon_cq_sched_t **)sched_hdl_p);
        return (status);
}


/*
 * hermon_ci_free_cq_sched()
 *    Free a CQ scheduling class resource
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_free_cq_sched(ibc_hca_hdl_t hca, ibc_sched_hdl_t sched_hdl)
{
        int     status;

        status = hermon_cq_sched_free((hermon_state_t *)hca,
            (hermon_cq_sched_t *)sched_hdl);
        return (status);
}

static ibt_status_t
hermon_ci_query_cq_handler_id(ibc_hca_hdl_t hca,
    ibt_cq_handler_id_t hid, ibt_cq_handler_attr_t *attrs)
{
        hermon_state_t          *state;

        state = (hermon_state_t *)hca;
        if (!HERMON_HID_VALID(state, hid))
                return (IBT_CQ_HID_INVALID);
        if (attrs == NULL)
                return (IBT_INVALID_PARAM);
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*attrs))
        attrs->cha_ih = state->hs_intrmsi_hdl[hid - 1];
        attrs->cha_dip = state->hs_dip;
        return (IBT_SUCCESS);
}

/*
 * hermon_ci_alloc_eec()
 *    Allocate an End-to-End context
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_eec(ibc_hca_hdl_t hca, ibc_eec_flags_t flags,
    ibt_eec_hdl_t ibt_eec, ibc_rdd_hdl_t rdd, ibc_eec_hdl_t *eec_p)
{
        /*
         * This is an unsupported interface for the Hermon driver.  This
         * interface is necessary to support Reliable Datagram (RD)
         * operations.  Hermon does not support RD.
         */
        return (IBT_NOT_SUPPORTED);
}


/*
 * hermon_ci_free_eec()
 *    Free an End-to-End context
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_free_eec(ibc_hca_hdl_t hca, ibc_eec_hdl_t eec)
{
        /*
         * This is an unsupported interface for the Hermon driver.  This
         * interface is necessary to support Reliable Datagram (RD)
         * operations.  Hermon does not support RD.
         */
        return (IBT_NOT_SUPPORTED);
}


/*
 * hermon_ci_query_eec()
 *    Query an End-to-End context
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_query_eec(ibc_hca_hdl_t hca, ibc_eec_hdl_t eec,
    ibt_eec_query_attr_t *attr_p)
{
        /*
         * This is an unsupported interface for the Hermon driver.  This
         * interface is necessary to support Reliable Datagram (RD)
         * operations.  Hermon does not support RD.
         */
        return (IBT_NOT_SUPPORTED);
}


/*
 * hermon_ci_modify_eec()
 *    Modify an End-to-End context
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_modify_eec(ibc_hca_hdl_t hca, ibc_eec_hdl_t eec,
    ibt_cep_modify_flags_t flags, ibt_eec_info_t *info_p)
{
        /*
         * This is an unsupported interface for the Hermon driver.  This
         * interface is necessary to support Reliable Datagram (RD)
         * operations.  Hermon does not support RD.
         */
        return (IBT_NOT_SUPPORTED);
}


/*
 * hermon_ci_register_mr()
 *    Prepare a virtually addressed Memory Region for use by an HCA
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_mr(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
    ibt_mr_attr_t *mr_attr, void *ibtl_reserved, ibc_mr_hdl_t *mr_p,
    ibt_mr_desc_t *mr_desc)
{
        hermon_mr_options_t     op;
        hermon_state_t          *state;
        hermon_pdhdl_t          pdhdl;
        hermon_mrhdl_t          mrhdl;
        int                     status;

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))

        ASSERT(mr_attr != NULL);
        ASSERT(mr_p != NULL);
        ASSERT(mr_desc != NULL);

        /*
         * Validate the access flags.  Both Remote Write and Remote Atomic
         * require the Local Write flag to be set
         */
        if (((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
            (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
            !(mr_attr->mr_flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
                return (IBT_MR_ACCESS_REQ_INVALID);
        }

        /* Grab the Hermon softstate pointer and PD handle */
        state = (hermon_state_t *)hca;
        pdhdl = (hermon_pdhdl_t)pd;

        /* Register the memory region */
        op.mro_bind_type   = state->hs_cfg_profile->cp_iommu_bypass;
        op.mro_bind_dmahdl = NULL;
        op.mro_bind_override_addr = 0;
        status = hermon_mr_register(state, pdhdl, mr_attr, &mrhdl,
            &op, HERMON_MPT_DMPT);
        if (status != DDI_SUCCESS) {
                return (status);
        }
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl))

        /* Fill in the mr_desc structure */
        mr_desc->md_vaddr = mrhdl->mr_bindinfo.bi_addr;
        mr_desc->md_lkey  = mrhdl->mr_lkey;
        /* Only set RKey if remote access was requested */
        if ((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
            (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
            (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ)) {
                mr_desc->md_rkey = mrhdl->mr_rkey;
        }

        /*
         * If region is mapped for streaming (i.e. noncoherent), then set
         * sync is required
         */
        mr_desc->md_sync_required = (mrhdl->mr_bindinfo.bi_flags &
            IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;

        /* Return the Hermon MR handle */
        *mr_p = (ibc_mr_hdl_t)mrhdl;

        return (IBT_SUCCESS);
}


/*
 * hermon_ci_register_buf()
 *    Prepare a Memory Region specified by buf structure for use by an HCA
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_buf(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
    ibt_smr_attr_t *attrp, struct buf *buf, void *ibtl_reserved,
    ibt_mr_hdl_t *mr_p, ibt_mr_desc_t *mr_desc)
{
        hermon_mr_options_t     op;
        hermon_state_t          *state;
        hermon_pdhdl_t          pdhdl;
        hermon_mrhdl_t          mrhdl;
        int                     status;
        ibt_mr_flags_t          flags = attrp->mr_flags;

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))

        ASSERT(mr_p != NULL);
        ASSERT(mr_desc != NULL);

        /*
         * Validate the access flags.  Both Remote Write and Remote Atomic
         * require the Local Write flag to be set
         */
        if (((flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
            (flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
            !(flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
                return (IBT_MR_ACCESS_REQ_INVALID);
        }

        /* Grab the Hermon softstate pointer and PD handle */
        state = (hermon_state_t *)hca;
        pdhdl = (hermon_pdhdl_t)pd;

        /* Register the memory region */
        op.mro_bind_type   = state->hs_cfg_profile->cp_iommu_bypass;
        op.mro_bind_dmahdl = NULL;
        op.mro_bind_override_addr = 0;
        status = hermon_mr_register_buf(state, pdhdl, attrp, buf,
            &mrhdl, &op, HERMON_MPT_DMPT);
        if (status != DDI_SUCCESS) {
                return (status);
        }
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl))

        /* Fill in the mr_desc structure */
        mr_desc->md_vaddr = mrhdl->mr_bindinfo.bi_addr;
        mr_desc->md_lkey  = mrhdl->mr_lkey;
        /* Only set RKey if remote access was requested */
        if ((flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
            (flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
            (flags & IBT_MR_ENABLE_REMOTE_READ)) {
                mr_desc->md_rkey = mrhdl->mr_rkey;
        }

        /*
         * If region is mapped for streaming (i.e. noncoherent), then set
         * sync is required
         */
        mr_desc->md_sync_required = (mrhdl->mr_bindinfo.bi_flags &
            IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;

        /* Return the Hermon MR handle */
        *mr_p = (ibc_mr_hdl_t)mrhdl;

        return (IBT_SUCCESS);
}


/*
 * hermon_ci_deregister_mr()
 *    Deregister a Memory Region from an HCA translation table
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_deregister_mr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr)
{
        hermon_state_t          *state;
        hermon_mrhdl_t          mrhdl;
        int                     status;

        /* Grab the Hermon softstate pointer */
        state = (hermon_state_t *)hca;
        mrhdl = (hermon_mrhdl_t)mr;

        /*
         * Deregister the memory region.
         */
        status = hermon_mr_deregister(state, &mrhdl, HERMON_MR_DEREG_ALL,
            HERMON_NOSLEEP);
        return (status);
}


/*
 * hermon_ci_query_mr()
 *    Retrieve information about a specified Memory Region
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_query_mr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr,
    ibt_mr_query_attr_t *mr_attr)
{
        hermon_state_t          *state;
        hermon_mrhdl_t          mrhdl;
        int                     status;

        ASSERT(mr_attr != NULL);

        /* Grab the Hermon softstate pointer and MR handle */
        state = (hermon_state_t *)hca;
        mrhdl = (hermon_mrhdl_t)mr;

        /* Query the memory region */
        status = hermon_mr_query(state, mrhdl, mr_attr);
        return (status);
}


/*
 * hermon_ci_register_shared_mr()
 *    Create a shared memory region matching an existing Memory Region
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_shared_mr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr,
    ibc_pd_hdl_t pd, ibt_smr_attr_t *mr_attr, void *ibtl_reserved,
    ibc_mr_hdl_t *mr_p, ibt_mr_desc_t *mr_desc)
{
        hermon_state_t          *state;
        hermon_pdhdl_t          pdhdl;
        hermon_mrhdl_t          mrhdl, mrhdl_new;
        int                     status;

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))

        ASSERT(mr_attr != NULL);
        ASSERT(mr_p != NULL);
        ASSERT(mr_desc != NULL);

        /*
         * Validate the access flags.  Both Remote Write and Remote Atomic
         * require the Local Write flag to be set
         */
        if (((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
            (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
            !(mr_attr->mr_flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
                return (IBT_MR_ACCESS_REQ_INVALID);
        }

        /* Grab the Hermon softstate pointer and handles */
        state = (hermon_state_t *)hca;
        pdhdl = (hermon_pdhdl_t)pd;
        mrhdl = (hermon_mrhdl_t)mr;

        /* Register the shared memory region */
        status = hermon_mr_register_shared(state, mrhdl, pdhdl, mr_attr,
            &mrhdl_new);
        if (status != DDI_SUCCESS) {
                return (status);
        }
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl_new))

        /* Fill in the mr_desc structure */
        mr_desc->md_vaddr = mrhdl_new->mr_bindinfo.bi_addr;
        mr_desc->md_lkey  = mrhdl_new->mr_lkey;
        /* Only set RKey if remote access was requested */
        if ((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
            (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
            (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ)) {
                mr_desc->md_rkey = mrhdl_new->mr_rkey;
        }

        /*
         * If shared region is mapped for streaming (i.e. noncoherent), then
         * set sync is required
         */
        mr_desc->md_sync_required = (mrhdl_new->mr_bindinfo.bi_flags &
            IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;

        /* Return the Hermon MR handle */
        *mr_p = (ibc_mr_hdl_t)mrhdl_new;

        return (IBT_SUCCESS);
}


/*
 * hermon_ci_reregister_mr()
 *    Modify the attributes of an existing Memory Region
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_reregister_mr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr, ibc_pd_hdl_t pd,
    ibt_mr_attr_t *mr_attr, void *ibtl_reserved, ibc_mr_hdl_t *mr_new,
    ibt_mr_desc_t *mr_desc)
{
        hermon_mr_options_t     op;
        hermon_state_t          *state;
        hermon_pdhdl_t          pdhdl;
        hermon_mrhdl_t          mrhdl, mrhdl_new;
        int                     status;

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))

        ASSERT(mr_attr != NULL);
        ASSERT(mr_new != NULL);
        ASSERT(mr_desc != NULL);

        /* Grab the Hermon softstate pointer, mrhdl, and pdhdl */
        state = (hermon_state_t *)hca;
        mrhdl = (hermon_mrhdl_t)mr;
        pdhdl = (hermon_pdhdl_t)pd;

        /* Reregister the memory region */
        op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
        status = hermon_mr_reregister(state, mrhdl, pdhdl, mr_attr,
            &mrhdl_new, &op);
        if (status != DDI_SUCCESS) {
                return (status);
        }
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl_new))

        /* Fill in the mr_desc structure */
        mr_desc->md_vaddr = mrhdl_new->mr_bindinfo.bi_addr;
        mr_desc->md_lkey  = mrhdl_new->mr_lkey;
        /* Only set RKey if remote access was requested */
        if ((mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
            (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
            (mr_attr->mr_flags & IBT_MR_ENABLE_REMOTE_READ)) {
                mr_desc->md_rkey = mrhdl_new->mr_rkey;
        }

        /*
         * If region is mapped for streaming (i.e. noncoherent), then set
         * sync is required
         */
        mr_desc->md_sync_required = (mrhdl_new->mr_bindinfo.bi_flags &
            IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;

        /* Return the Hermon MR handle */
        *mr_new = (ibc_mr_hdl_t)mrhdl_new;

        return (IBT_SUCCESS);
}


/*
 * hermon_ci_reregister_buf()
 *    Modify the attributes of an existing Memory Region
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_reregister_buf(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr, ibc_pd_hdl_t pd,
    ibt_smr_attr_t *attrp, struct buf *buf, void *ibtl_reserved,
    ibc_mr_hdl_t *mr_new, ibt_mr_desc_t *mr_desc)
{
        hermon_mr_options_t     op;
        hermon_state_t          *state;
        hermon_pdhdl_t          pdhdl;
        hermon_mrhdl_t          mrhdl, mrhdl_new;
        int                     status;
        ibt_mr_flags_t          flags = attrp->mr_flags;

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))

        ASSERT(mr_new != NULL);
        ASSERT(mr_desc != NULL);

        /* Grab the Hermon softstate pointer, mrhdl, and pdhdl */
        state = (hermon_state_t *)hca;
        mrhdl = (hermon_mrhdl_t)mr;
        pdhdl = (hermon_pdhdl_t)pd;

        /* Reregister the memory region */
        op.mro_bind_type = state->hs_cfg_profile->cp_iommu_bypass;
        status = hermon_mr_reregister_buf(state, mrhdl, pdhdl, attrp, buf,
            &mrhdl_new, &op);
        if (status != DDI_SUCCESS) {
                return (status);
        }
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl_new))

        /* Fill in the mr_desc structure */
        mr_desc->md_vaddr = mrhdl_new->mr_bindinfo.bi_addr;
        mr_desc->md_lkey  = mrhdl_new->mr_lkey;
        /* Only set RKey if remote access was requested */
        if ((flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
            (flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
            (flags & IBT_MR_ENABLE_REMOTE_READ)) {
                mr_desc->md_rkey = mrhdl_new->mr_rkey;
        }

        /*
         * If region is mapped for streaming (i.e. noncoherent), then set
         * sync is required
         */
        mr_desc->md_sync_required = (mrhdl_new->mr_bindinfo.bi_flags &
            IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;

        /* Return the Hermon MR handle */
        *mr_new = (ibc_mr_hdl_t)mrhdl_new;

        return (IBT_SUCCESS);
}

/*
 * hermon_ci_sync_mr()
 *    Synchronize access to a Memory Region
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_sync_mr(ibc_hca_hdl_t hca, ibt_mr_sync_t *mr_segs, size_t num_segs)
{
        hermon_state_t          *state;
        int                     status;

        ASSERT(mr_segs != NULL);

        /* Grab the Hermon softstate pointer */
        state = (hermon_state_t *)hca;

        /* Sync the memory region */
        status = hermon_mr_sync(state, mr_segs, num_segs);
        return (status);
}


/*
 * hermon_ci_alloc_mw()
 *    Allocate a Memory Window
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_alloc_mw(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd, ibt_mw_flags_t flags,
    ibc_mw_hdl_t *mw_p, ibt_rkey_t *rkey_p)
{
        hermon_state_t          *state;
        hermon_pdhdl_t          pdhdl;
        hermon_mwhdl_t          mwhdl;
        int                     status;

        ASSERT(mw_p != NULL);
        ASSERT(rkey_p != NULL);

        /* Grab the Hermon softstate pointer and PD handle */
        state = (hermon_state_t *)hca;
        pdhdl = (hermon_pdhdl_t)pd;

        /* Allocate the memory window */
        status = hermon_mw_alloc(state, pdhdl, flags, &mwhdl);
        if (status != DDI_SUCCESS) {
                return (status);
        }
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mwhdl))

        /* Return the MW handle and RKey */
        *mw_p = (ibc_mw_hdl_t)mwhdl;
        *rkey_p = mwhdl->mr_rkey;

        return (IBT_SUCCESS);
}


/*
 * hermon_ci_free_mw()
 *    Free a Memory Window
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_free_mw(ibc_hca_hdl_t hca, ibc_mw_hdl_t mw)
{
        hermon_state_t          *state;
        hermon_mwhdl_t          mwhdl;
        int                     status;

        /* Grab the Hermon softstate pointer and MW handle */
        state = (hermon_state_t *)hca;
        mwhdl = (hermon_mwhdl_t)mw;

        /* Free the memory window */
        status = hermon_mw_free(state, &mwhdl, HERMON_NOSLEEP);
        return (status);
}


/*
 * hermon_ci_query_mw()
 *    Return the attributes of the specified Memory Window
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_query_mw(ibc_hca_hdl_t hca, ibc_mw_hdl_t mw,
    ibt_mw_query_attr_t *mw_attr_p)
{
        hermon_mwhdl_t          mwhdl;

        ASSERT(mw_attr_p != NULL);

        /* Query the memory window pointer and fill in the return values */
        mwhdl = (hermon_mwhdl_t)mw;
        mutex_enter(&mwhdl->mr_lock);
        mw_attr_p->mw_pd   = (ibc_pd_hdl_t)mwhdl->mr_pdhdl;
        mw_attr_p->mw_rkey = mwhdl->mr_rkey;
        mutex_exit(&mwhdl->mr_lock);

        return (IBT_SUCCESS);
}


/*
 * hermon_ci_register_dma_mr()
 *    Allocate a memory region that maps physical addresses.
 *    Context: Can be called only from user or kernel context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_dma_mr(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
    ibt_dmr_attr_t *mr_attr, void *ibtl_reserved, ibc_mr_hdl_t *mr_p,
    ibt_mr_desc_t *mr_desc)
{
        hermon_state_t          *state;
        hermon_pdhdl_t          pdhdl;
        hermon_mrhdl_t          mrhdl;
        int                     status;

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mr_desc))

        ASSERT(mr_attr != NULL);
        ASSERT(mr_p != NULL);
        ASSERT(mr_desc != NULL);

        /*
         * Validate the access flags.  Both Remote Write and Remote Atomic
         * require the Local Write flag to be set
         */
        if (((mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
            (mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
            !(mr_attr->dmr_flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
                return (IBT_MR_ACCESS_REQ_INVALID);
        }

        /* Grab the Hermon softstate pointer and PD handle */
        state = (hermon_state_t *)hca;
        pdhdl = (hermon_pdhdl_t)pd;

        status = hermon_dma_mr_register(state, pdhdl, mr_attr, &mrhdl);
        if (status != DDI_SUCCESS) {
                return (status);
        }
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl))

        /* Fill in the mr_desc structure */
        mr_desc->md_vaddr = mr_attr->dmr_paddr;
        mr_desc->md_lkey  = mrhdl->mr_lkey;
        /* Only set RKey if remote access was requested */
        if ((mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC) ||
            (mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
            (mr_attr->dmr_flags & IBT_MR_ENABLE_REMOTE_READ)) {
                mr_desc->md_rkey = mrhdl->mr_rkey;
        }

        /*
         * If region is mapped for streaming (i.e. noncoherent), then set
         * sync is required
         */
        mr_desc->md_sync_required = B_FALSE;

        /* Return the Hermon MR handle */
        *mr_p = (ibc_mr_hdl_t)mrhdl;

        return (IBT_SUCCESS);
}


/*
 * hermon_ci_attach_mcg()
 *    Attach a Queue Pair to a Multicast Group
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_attach_mcg(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp, ib_gid_t gid,
    ib_lid_t lid)
{
        hermon_state_t          *state;
        hermon_qphdl_t          qphdl;
        int                     status;

        /* Grab the Hermon softstate pointer and QP handles */
        state = (hermon_state_t *)hca;
        qphdl = (hermon_qphdl_t)qp;

        /* Attach the QP to the multicast group */
        status = hermon_mcg_attach(state, qphdl, gid, lid);
        return (status);
}


/*
 * hermon_ci_detach_mcg()
 *    Detach a Queue Pair to a Multicast Group
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_detach_mcg(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp, ib_gid_t gid,
    ib_lid_t lid)
{
        hermon_state_t          *state;
        hermon_qphdl_t          qphdl;
        int                     status;

        /* Grab the Hermon softstate pointer and QP handle */
        state = (hermon_state_t *)hca;
        qphdl = (hermon_qphdl_t)qp;

        /* Detach the QP from the multicast group */
        status = hermon_mcg_detach(state, qphdl, gid, lid);
        return (status);
}


/*
 * hermon_ci_post_send()
 *    Post send work requests to the send queue on the specified QP
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_post_send(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp, ibt_send_wr_t *wr_p,
    uint_t num_wr, uint_t *num_posted_p)
{
        hermon_state_t          *state;
        hermon_qphdl_t          qphdl;
        int                     status;

        ASSERT(wr_p != NULL);
        ASSERT(num_wr != 0);

        /* Grab the Hermon softstate pointer and QP handle */
        state = (hermon_state_t *)hca;
        qphdl = (hermon_qphdl_t)qp;

        /* Post the send WQEs */
        status = hermon_post_send(state, qphdl, wr_p, num_wr, num_posted_p);
        return (status);
}


/*
 * hermon_ci_post_recv()
 *    Post receive work requests to the receive queue on the specified QP
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_post_recv(ibc_hca_hdl_t hca, ibc_qp_hdl_t qp, ibt_recv_wr_t *wr_p,
    uint_t num_wr, uint_t *num_posted_p)
{
        hermon_state_t          *state;
        hermon_qphdl_t          qphdl;
        int                     status;

        ASSERT(wr_p != NULL);
        ASSERT(num_wr != 0);

        state = (hermon_state_t *)hca;
        qphdl = (hermon_qphdl_t)qp;

        /* Post the receive WQEs */
        status = hermon_post_recv(state, qphdl, wr_p, num_wr, num_posted_p);
        return (status);
}


/*
 * hermon_ci_poll_cq()
 *    Poll for a work request completion
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_poll_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq, ibt_wc_t *wc_p,
    uint_t num_wc, uint_t *num_polled)
{
        hermon_state_t          *state;
        hermon_cqhdl_t          cqhdl;
        int                     status;

        ASSERT(wc_p != NULL);

        /* Check for valid num_wc field */
        if (num_wc == 0) {
                return (IBT_INVALID_PARAM);
        }

        /* Grab the Hermon softstate pointer and CQ handle */
        state = (hermon_state_t *)hca;
        cqhdl = (hermon_cqhdl_t)cq;

        /* Poll for work request completions */
        status = hermon_cq_poll(state, cqhdl, wc_p, num_wc, num_polled);
        return (status);
}


/*
 * hermon_ci_notify_cq()
 *    Enable notification events on the specified CQ
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_notify_cq(ibc_hca_hdl_t hca, ibc_cq_hdl_t cq_hdl,
    ibt_cq_notify_flags_t flags)
{
        hermon_state_t          *state;
        hermon_cqhdl_t          cqhdl;
        int                     status;

        /* Grab the Hermon softstate pointer and CQ handle */
        state = (hermon_state_t *)hca;
        cqhdl = (hermon_cqhdl_t)cq_hdl;

        /* Enable the CQ notification */
        status = hermon_cq_notify(state, cqhdl, flags);
        return (status);
}

/*
 * hermon_ci_ci_data_in()
 *    Exchange CI-specific data.
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_ci_data_in(ibc_hca_hdl_t hca, ibt_ci_data_flags_t flags,
    ibt_object_type_t object, void *ibc_object_handle, void *data_p,
    size_t data_sz)
{
        hermon_state_t          *state;
        int                     status;

        /* Grab the Hermon softstate pointer */
        state = (hermon_state_t *)hca;

        /* Get the Hermon userland mapping information */
        status = hermon_umap_ci_data_in(state, flags, object,
            ibc_object_handle, data_p, data_sz);
        return (status);
}

/*
 * hermon_ci_ci_data_out()
 *    Exchange CI-specific data.
 *    Context: Can be called only from user or kernel context.
 */
static ibt_status_t
hermon_ci_ci_data_out(ibc_hca_hdl_t hca, ibt_ci_data_flags_t flags,
    ibt_object_type_t object, void *ibc_object_handle, void *data_p,
    size_t data_sz)
{
        hermon_state_t          *state;
        int                     status;

        /* Grab the Hermon softstate pointer */
        state = (hermon_state_t *)hca;

        /* Get the Hermon userland mapping information */
        status = hermon_umap_ci_data_out(state, flags, object,
            ibc_object_handle, data_p, data_sz);
        return (status);
}


/*
 * hermon_ci_alloc_srq()
 *    Allocate a Shared Receive Queue (SRQ)
 *    Context: Can be called only from user or kernel context
 */
static ibt_status_t
hermon_ci_alloc_srq(ibc_hca_hdl_t hca, ibt_srq_flags_t flags,
    ibt_srq_hdl_t ibt_srq, ibc_pd_hdl_t pd, ibt_srq_sizes_t *sizes,
    ibc_srq_hdl_t *ibc_srq_p, ibt_srq_sizes_t *ret_sizes_p)
{
        hermon_state_t          *state;
        hermon_pdhdl_t          pdhdl;
        hermon_srqhdl_t         srqhdl;
        hermon_srq_info_t       srqinfo;
        int                     status;

        state = (hermon_state_t *)hca;
        pdhdl = (hermon_pdhdl_t)pd;

        srqinfo.srqi_ibt_srqhdl = ibt_srq;
        srqinfo.srqi_pd         = pdhdl;
        srqinfo.srqi_sizes      = sizes;
        srqinfo.srqi_real_sizes = ret_sizes_p;
        srqinfo.srqi_srqhdl     = &srqhdl;
        srqinfo.srqi_flags      = flags;

        status = hermon_srq_alloc(state, &srqinfo, HERMON_NOSLEEP);
        if (status != DDI_SUCCESS) {
                return (status);
        }

        *ibc_srq_p = (ibc_srq_hdl_t)srqhdl;

        return (IBT_SUCCESS);
}

/*
 * hermon_ci_free_srq()
 *    Free a Shared Receive Queue (SRQ)
 *    Context: Can be called only from user or kernel context
 */
static ibt_status_t
hermon_ci_free_srq(ibc_hca_hdl_t hca, ibc_srq_hdl_t srq)
{
        hermon_state_t  *state;
        hermon_srqhdl_t srqhdl;
        int             status;

        state = (hermon_state_t *)hca;

        /* Check for valid SRQ handle pointer */
        if (srq == NULL) {
                return (IBT_SRQ_HDL_INVALID);
        }

        srqhdl = (hermon_srqhdl_t)srq;

        /* Free the SRQ */
        status = hermon_srq_free(state, &srqhdl, HERMON_NOSLEEP);
        return (status);
}

/*
 * hermon_ci_query_srq()
 *    Query properties of a Shared Receive Queue (SRQ)
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_query_srq(ibc_hca_hdl_t hca, ibc_srq_hdl_t srq, ibc_pd_hdl_t *pd_p,
    ibt_srq_sizes_t *sizes_p, uint_t *limit_p)
{
        hermon_srqhdl_t srqhdl;

        srqhdl = (hermon_srqhdl_t)srq;

        mutex_enter(&srqhdl->srq_lock);
        if (srqhdl->srq_state == HERMON_SRQ_STATE_ERROR) {
                mutex_exit(&srqhdl->srq_lock);
                return (IBT_SRQ_ERROR_STATE);
        }

        *pd_p   = (ibc_pd_hdl_t)srqhdl->srq_pdhdl;
        sizes_p->srq_wr_sz = srqhdl->srq_real_sizes.srq_wr_sz - 1;
        sizes_p->srq_sgl_sz = srqhdl->srq_real_sizes.srq_sgl_sz;
        mutex_exit(&srqhdl->srq_lock);
        *limit_p  = 0;

        return (IBT_SUCCESS);
}

/*
 * hermon_ci_modify_srq()
 *    Modify properties of a Shared Receive Queue (SRQ)
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_modify_srq(ibc_hca_hdl_t hca, ibc_srq_hdl_t srq,
    ibt_srq_modify_flags_t flags, uint_t size, uint_t limit, uint_t *ret_size_p)
{
        hermon_state_t  *state;
        hermon_srqhdl_t srqhdl;
        uint_t          resize_supported, cur_srq_size;
        int             status;

        state = (hermon_state_t *)hca;
        srqhdl = (hermon_srqhdl_t)srq;

        /*
         * Check Error State of SRQ.
         * Also, while we are holding the lock we save away the current SRQ
         * size for later use.
         */
        mutex_enter(&srqhdl->srq_lock);
        cur_srq_size = srqhdl->srq_wq_bufsz;
        if (srqhdl->srq_state == HERMON_SRQ_STATE_ERROR) {
                mutex_exit(&srqhdl->srq_lock);
                return (IBT_SRQ_ERROR_STATE);
        }
        mutex_exit(&srqhdl->srq_lock);

        /*
         * Setting the limit watermark is not currently supported.  This is a
         * hermon hardware (firmware) limitation.  We return NOT_SUPPORTED here,
         * and have the limit code commented out for now.
         *
         * XXX If we enable the limit watermark support, we need to do checks
         * and set the 'srq->srq_wr_limit' here, instead of returning not
         * supported.  The 'hermon_srq_modify' operation below is for resizing
         * the SRQ only, the limit work should be done here.  If this is
         * changed to use the 'limit' field, the 'ARGSUSED' comment for this
         * function should also be removed at that time.
         */
        if (flags & IBT_SRQ_SET_LIMIT) {
                return (IBT_NOT_SUPPORTED);
        }

        /*
         * Check the SET_SIZE flag.  If not set, we simply return success here.
         * However if it is set, we check if resize is supported and only then
         * do we continue on with our resize processing.
         */
        if (!(flags & IBT_SRQ_SET_SIZE)) {
                return (IBT_SUCCESS);
        }

        resize_supported = state->hs_ibtfinfo.hca_attr->hca_flags &
            IBT_HCA_RESIZE_SRQ;

        if ((flags & IBT_SRQ_SET_SIZE) && !resize_supported) {
                return (IBT_NOT_SUPPORTED);
        }

        /*
         * We do not support resizing an SRQ to be smaller than it's current
         * size.  If a smaller (or equal) size is requested, then we simply
         * return success, and do nothing.
         */
        if (size <= cur_srq_size) {
                *ret_size_p = cur_srq_size;
                return (IBT_SUCCESS);
        }

        status = hermon_srq_modify(state, srqhdl, size, ret_size_p,
            HERMON_NOSLEEP);
        if (status != DDI_SUCCESS) {
                /* Set return value to current SRQ size */
                *ret_size_p = cur_srq_size;
                return (status);
        }

        return (IBT_SUCCESS);
}

/*
 * hermon_ci_post_srq()
 *    Post a Work Request to the specified Shared Receive Queue (SRQ)
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_post_srq(ibc_hca_hdl_t hca, ibc_srq_hdl_t srq,
    ibt_recv_wr_t *wr, uint_t num_wr, uint_t *num_posted_p)
{
        hermon_state_t  *state;
        hermon_srqhdl_t srqhdl;
        int             status;

        state = (hermon_state_t *)hca;
        srqhdl = (hermon_srqhdl_t)srq;

        status = hermon_post_srq(state, srqhdl, wr, num_wr, num_posted_p);
        return (status);
}

/* Address translation */

struct ibc_ma_s {
        int                     h_ma_addr_list_len;
        void                    *h_ma_addr_list;
        ddi_dma_handle_t        h_ma_dmahdl;
        ddi_dma_handle_t        h_ma_list_hdl;
        ddi_acc_handle_t        h_ma_list_acc_hdl;
        size_t                  h_ma_real_len;
        caddr_t                 h_ma_kaddr;
        ibt_phys_addr_t         h_ma_list_cookie;
};

static ibt_status_t
hermon_map_mem_area_fmr(ibc_hca_hdl_t hca, ibt_va_attr_t *va_attrs,
    uint_t list_len, ibt_pmr_attr_t *pmr, ibc_ma_hdl_t *ma_hdl_p)
{
        int                     status;
        ibt_status_t            ibt_status;
        ibc_ma_hdl_t            ma_hdl;
        ib_memlen_t             len;
        ddi_dma_attr_t          dma_attr;
        uint_t                  cookie_cnt;
        ddi_dma_cookie_t        dmacookie;
        hermon_state_t          *state;
        uint64_t                *kaddr;
        uint64_t                addr, endaddr, pagesize;
        int                     i, kmflag;
        int                     (*callback)(caddr_t);

        if ((va_attrs->va_flags & IBT_VA_BUF) == 0) {
                return (IBT_NOT_SUPPORTED);     /* XXX - not yet implemented */
        }

        state = (hermon_state_t *)hca;
        hermon_dma_attr_init(state, &dma_attr);
        if (va_attrs->va_flags & IBT_VA_NOSLEEP) {
                kmflag = KM_NOSLEEP;
                callback = DDI_DMA_DONTWAIT;
        } else {
                kmflag = KM_SLEEP;
                callback = DDI_DMA_SLEEP;
        }

        ma_hdl = kmem_zalloc(sizeof (*ma_hdl), kmflag);
        if (ma_hdl == NULL) {
                return (IBT_INSUFF_RESOURCE);
        }
#ifdef  __sparc
        if (state->hs_cfg_profile->cp_iommu_bypass == HERMON_BINDMEM_BYPASS)
                dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;

        if (hermon_kernel_data_ro == HERMON_RO_ENABLED)
                dma_attr.dma_attr_flags |= DDI_DMA_RELAXED_ORDERING;
#endif

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ma_hdl))
        status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
            callback, NULL, &ma_hdl->h_ma_dmahdl);
        if (status != DDI_SUCCESS) {
                kmem_free(ma_hdl, sizeof (*ma_hdl));
                return (IBT_INSUFF_RESOURCE);
        }
        status = ddi_dma_buf_bind_handle(ma_hdl->h_ma_dmahdl,
            va_attrs->va_buf, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
            callback, NULL, &dmacookie, &cookie_cnt);
        if (status != DDI_DMA_MAPPED) {
                status = ibc_get_ci_failure(0);
                goto marea_fail3;
        }

        ma_hdl->h_ma_real_len = list_len * sizeof (ibt_phys_addr_t);
        ma_hdl->h_ma_kaddr = kmem_zalloc(ma_hdl->h_ma_real_len, kmflag);
        if (ma_hdl->h_ma_kaddr == NULL) {
                ibt_status = IBT_INSUFF_RESOURCE;
                goto marea_fail4;
        }

        i = 0;
        len = 0;
        pagesize = PAGESIZE;
        kaddr = (uint64_t *)(void *)ma_hdl->h_ma_kaddr;
        while (cookie_cnt-- > 0) {
                addr    = dmacookie.dmac_laddress;
                len     += dmacookie.dmac_size;
                endaddr = addr + (dmacookie.dmac_size - 1);
                addr    = addr & ~(pagesize - 1);
                while (addr <= endaddr) {
                        if (i >= list_len) {
                                status = IBT_PBL_TOO_SMALL;
                                goto marea_fail5;
                        }
                        kaddr[i] = htonll(addr | HERMON_MTT_ENTRY_PRESENT);
                        i++;
                        addr += pagesize;
                        if (addr == 0) {
                                static int do_once = 1;
                                _NOTE(SCHEME_PROTECTS_DATA("safe sharing",
                                    do_once))
                                if (do_once) {
                                        do_once = 0;
                                        cmn_err(CE_NOTE, "probable error in "
                                            "dma_cookie address: map_mem_area");
                                }
                                break;
                        }
                }
                if (cookie_cnt != 0)
                        ddi_dma_nextcookie(ma_hdl->h_ma_dmahdl, &dmacookie);
        }

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pmr))
        pmr->pmr_addr_list = (ibt_phys_addr_t *)(void *)ma_hdl->h_ma_kaddr;
        pmr->pmr_iova = va_attrs->va_vaddr;
        pmr->pmr_len = len;
        pmr->pmr_offset = va_attrs->va_vaddr & PAGEOFFSET;
        pmr->pmr_buf_sz = PAGESHIFT;    /* PRM says "Page Sice", but... */
        pmr->pmr_num_buf = i;
        pmr->pmr_ma = ma_hdl;

        *ma_hdl_p = ma_hdl;
        return (IBT_SUCCESS);

marea_fail5:
        kmem_free(ma_hdl->h_ma_kaddr, ma_hdl->h_ma_real_len);
marea_fail4:
        status = ddi_dma_unbind_handle(ma_hdl->h_ma_dmahdl);
marea_fail3:
        ddi_dma_free_handle(&ma_hdl->h_ma_dmahdl);
        kmem_free(ma_hdl, sizeof (*ma_hdl));
        *ma_hdl_p = NULL;
        return (ibt_status);
}

/*
 * hermon_ci_map_mem_area()
 *    Context: Can be called from user or base context.
 *
 *      Creates the memory mapping suitable for a subsequent posting of an
 *      FRWR work request.  All the info about the memory area for the
 *      FRWR work request (wr member of "union ibt_reg_req_u") is filled
 *      such that the client only needs to point wr.rc.rcwr.reg_pmr to it,
 *      and then fill in the additional information only it knows.
 *
 *      Alternatively, creates the memory mapping for FMR.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_map_mem_area(ibc_hca_hdl_t hca, ibt_va_attr_t *va_attrs,
    void *ibtl_reserved, uint_t list_len, ibt_reg_req_t *reg_req,
    ibc_ma_hdl_t *ma_hdl_p)
{
        ibt_status_t            ibt_status;
        int                     status;
        ibc_ma_hdl_t            ma_hdl;
        ibt_wr_reg_pmr_t        *pmr;
        ib_memlen_t             len;
        ddi_dma_attr_t          dma_attr;
        ddi_dma_handle_t        khdl;
        uint_t                  cookie_cnt;
        ddi_dma_cookie_t        dmacookie, kcookie;
        hermon_state_t          *state;
        uint64_t                *kaddr;
        uint64_t                addr, endaddr, pagesize, kcookie_paddr;
        int                     i, j, kmflag;
        int                     (*callback)(caddr_t);

        if (va_attrs->va_flags & (IBT_VA_FMR | IBT_VA_REG_FN)) {
                /* delegate FMR and Physical Register to other function */
                return (hermon_map_mem_area_fmr(hca, va_attrs, list_len,
                    &reg_req->fn_arg, ma_hdl_p));
        }

        /* FRWR */

        state = (hermon_state_t *)hca;
        if (!(state->hs_ibtfinfo.hca_attr->hca_flags2 & IBT_HCA2_MEM_MGT_EXT))
                return (IBT_NOT_SUPPORTED);
        hermon_dma_attr_init(state, &dma_attr);
#ifdef  __sparc
        if (state->hs_cfg_profile->cp_iommu_bypass == HERMON_BINDMEM_BYPASS)
                dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;

        if (hermon_kernel_data_ro == HERMON_RO_ENABLED)
                dma_attr.dma_attr_flags |= DDI_DMA_RELAXED_ORDERING;
#endif
        if (va_attrs->va_flags & IBT_VA_NOSLEEP) {
                kmflag = KM_NOSLEEP;
                callback = DDI_DMA_DONTWAIT;
        } else {
                kmflag = KM_SLEEP;
                callback = DDI_DMA_SLEEP;
        }

        ma_hdl = kmem_zalloc(sizeof (*ma_hdl), kmflag);
        if (ma_hdl == NULL) {
                return (IBT_INSUFF_RESOURCE);
        }
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*ma_hdl))

        status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
            callback, NULL, &ma_hdl->h_ma_dmahdl);
        if (status != DDI_SUCCESS) {
                ibt_status = IBT_INSUFF_RESOURCE;
                goto marea_fail0;
        }
        dma_attr.dma_attr_align = 64;   /* as per PRM */
        status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
            callback, NULL, &ma_hdl->h_ma_list_hdl);
        if (status != DDI_SUCCESS) {
                ibt_status = IBT_INSUFF_RESOURCE;
                goto marea_fail1;
        }
        /*
         * Entries in the list in the last slot on each page cannot be used,
         * so 1 extra ibt_phys_addr_t is allocated per page.  We add 1 more
         * to deal with the possibility of a less than 1 page allocation
         * across a page boundary.
         */
        status = ddi_dma_mem_alloc(ma_hdl->h_ma_list_hdl, (list_len + 1 +
            list_len / (HERMON_PAGESIZE / sizeof (ibt_phys_addr_t))) *
            sizeof (ibt_phys_addr_t),
            &state->hs_reg_accattr, DDI_DMA_CONSISTENT, callback, NULL,
            &ma_hdl->h_ma_kaddr, &ma_hdl->h_ma_real_len,
            &ma_hdl->h_ma_list_acc_hdl);
        if (status != DDI_SUCCESS) {
                ibt_status = IBT_INSUFF_RESOURCE;
                goto marea_fail2;
        }
        status = ddi_dma_addr_bind_handle(ma_hdl->h_ma_list_hdl, NULL,
            ma_hdl->h_ma_kaddr, ma_hdl->h_ma_real_len, DDI_DMA_RDWR |
            DDI_DMA_CONSISTENT, callback, NULL,
            &kcookie, &cookie_cnt);
        if (status != DDI_SUCCESS) {
                ibt_status = IBT_INSUFF_RESOURCE;
                goto marea_fail3;
        }
        if ((kcookie.dmac_laddress & 0x3f) != 0) {
                cmn_err(CE_NOTE, "64-byte alignment assumption wrong");
                ibt_status = ibc_get_ci_failure(0);
                goto marea_fail4;
        }
        ma_hdl->h_ma_list_cookie.p_laddr = kcookie.dmac_laddress;

        if (va_attrs->va_flags & IBT_VA_BUF) {
                status = ddi_dma_buf_bind_handle(ma_hdl->h_ma_dmahdl,
                    va_attrs->va_buf, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
                    callback, NULL, &dmacookie, &cookie_cnt);
        } else {
                status = ddi_dma_addr_bind_handle(ma_hdl->h_ma_dmahdl,
                    va_attrs->va_as, (caddr_t)(uintptr_t)va_attrs->va_vaddr,
                    va_attrs->va_len, DDI_DMA_RDWR | DDI_DMA_CONSISTENT,
                    callback, NULL, &dmacookie, &cookie_cnt);
        }
        if (status != DDI_DMA_MAPPED) {
                ibt_status = ibc_get_ci_failure(0);
                goto marea_fail4;
        }
        i = 0;  /* count the number of pbl entries */
        j = 0;  /* count the number of links to next HERMON_PAGE */
        len = 0;
        pagesize = PAGESIZE;
        kaddr = (uint64_t *)(void *)ma_hdl->h_ma_kaddr;
        kcookie.dmac_size += kcookie.dmac_laddress & HERMON_PAGEOFFSET;
        kcookie_paddr = kcookie.dmac_laddress & HERMON_PAGEMASK;
        khdl = ma_hdl->h_ma_list_hdl;
        while (cookie_cnt-- > 0) {
                addr    = dmacookie.dmac_laddress;
                len     += dmacookie.dmac_size;
                endaddr = addr + (dmacookie.dmac_size - 1);
                addr    = addr & ~(pagesize - 1);
                while (addr <= endaddr) {
                        if (i >= list_len) {
                                ibt_status = IBT_PBL_TOO_SMALL;
                                goto marea_fail5;
                        }
                        /* Deal with last entry on page. */
                        if (!((uintptr_t)&kaddr[i+j+1] & HERMON_PAGEOFFSET)) {
                                if (kcookie.dmac_size > HERMON_PAGESIZE) {
                                        kcookie_paddr += HERMON_PAGESIZE;
                                        kcookie.dmac_size -= HERMON_PAGESIZE;
                                } else {
                                        ddi_dma_nextcookie(khdl, &kcookie);
                                        kcookie_paddr = kcookie.dmac_laddress;
                                }
                                kaddr[i+j] = htonll(kcookie_paddr);
                                j++;
                        }
                        kaddr[i+j] = htonll(addr | HERMON_MTT_ENTRY_PRESENT);
                        i++;
                        addr += pagesize;
                        if (addr == 0) {
                                static int do_once = 1;
                                _NOTE(SCHEME_PROTECTS_DATA("safe sharing",
                                    do_once))
                                if (do_once) {
                                        do_once = 0;
                                        cmn_err(CE_NOTE, "probable error in "
                                            "dma_cookie address: map_mem_area");
                                }
                                break;
                        }
                }
                if (cookie_cnt != 0)
                        ddi_dma_nextcookie(ma_hdl->h_ma_dmahdl, &dmacookie);
        }

        pmr = &reg_req->wr;
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*pmr))
        pmr->pmr_len = len;
        pmr->pmr_offset = va_attrs->va_vaddr & PAGEOFFSET;
        pmr->pmr_buf_sz = PAGESHIFT;    /* PRM says "Page Size", but... */
        pmr->pmr_num_buf = i;
        pmr->pmr_addr_list = &ma_hdl->h_ma_list_cookie;

        *ma_hdl_p = ma_hdl;
        return (IBT_SUCCESS);

marea_fail5:
        status = ddi_dma_unbind_handle(ma_hdl->h_ma_dmahdl);
        if (status != DDI_SUCCESS)
                HERMON_WARNING(state, "failed to unbind DMA mapping");
marea_fail4:
        status = ddi_dma_unbind_handle(ma_hdl->h_ma_list_hdl);
        if (status != DDI_SUCCESS)
                HERMON_WARNING(state, "failed to unbind DMA mapping");
marea_fail3:
        ddi_dma_mem_free(&ma_hdl->h_ma_list_acc_hdl);
marea_fail2:
        ddi_dma_free_handle(&ma_hdl->h_ma_list_hdl);
marea_fail1:
        ddi_dma_free_handle(&ma_hdl->h_ma_dmahdl);
marea_fail0:
        kmem_free(ma_hdl, sizeof (*ma_hdl));
        *ma_hdl_p = NULL;
        return (ibt_status);
}

/*
 * hermon_ci_unmap_mem_area()
 * Unmap the memory area
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_unmap_mem_area(ibc_hca_hdl_t hca, ibc_ma_hdl_t ma_hdl)
{
        int                     status;
        hermon_state_t          *state;

        if (ma_hdl == NULL) {
                return (IBT_MA_HDL_INVALID);
        }
        state = (hermon_state_t *)hca;
        if (ma_hdl->h_ma_list_hdl != NULL) {
                status = ddi_dma_unbind_handle(ma_hdl->h_ma_list_hdl);
                if (status != DDI_SUCCESS)
                        HERMON_WARNING(state, "failed to unbind DMA mapping");
                ddi_dma_mem_free(&ma_hdl->h_ma_list_acc_hdl);
                ddi_dma_free_handle(&ma_hdl->h_ma_list_hdl);
        } else {
                kmem_free(ma_hdl->h_ma_kaddr, ma_hdl->h_ma_real_len);
        }
        status = ddi_dma_unbind_handle(ma_hdl->h_ma_dmahdl);
        if (status != DDI_SUCCESS)
                HERMON_WARNING(state, "failed to unbind DMA mapping");
        ddi_dma_free_handle(&ma_hdl->h_ma_dmahdl);
        kmem_free(ma_hdl, sizeof (*ma_hdl));
        return (IBT_SUCCESS);
}

struct ibc_mi_s {
        int                     imh_len;
        ddi_dma_handle_t        imh_dmahandle[1];
};
_NOTE(SCHEME_PROTECTS_DATA("safe sharing",
    ibc_mi_s::imh_len
    ibc_mi_s::imh_dmahandle))


/*
 * hermon_ci_map_mem_iov()
 * Map the memory
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_map_mem_iov(ibc_hca_hdl_t hca, ibt_iov_attr_t *iov_attr,
    ibt_all_wr_t *wr, ibc_mi_hdl_t *mi_hdl_p)
{
        int                     status;
        int                     i, j, nds, max_nds;
        uint_t                  len;
        ibt_status_t            ibt_status;
        ddi_dma_handle_t        dmahdl;
        ddi_dma_cookie_t        dmacookie;
        ddi_dma_attr_t          dma_attr;
        uint_t                  cookie_cnt;
        ibc_mi_hdl_t            mi_hdl;
        ibt_lkey_t              rsvd_lkey;
        ibt_wr_ds_t             *sgl;
        hermon_state_t          *state;
        int                     kmflag;
        int                     (*callback)(caddr_t);

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*wr))

        state = (hermon_state_t *)hca;
        hermon_dma_attr_init(state, &dma_attr);
#ifdef  __sparc
        if (state->hs_cfg_profile->cp_iommu_bypass == HERMON_BINDMEM_BYPASS)
                dma_attr.dma_attr_flags = DDI_DMA_FORCE_PHYSICAL;

        if (hermon_kernel_data_ro == HERMON_RO_ENABLED)
                dma_attr.dma_attr_flags |= DDI_DMA_RELAXED_ORDERING;
#endif

        nds = 0;
        max_nds = iov_attr->iov_wr_nds;
        if (iov_attr->iov_lso_hdr_sz)
                max_nds -= (iov_attr->iov_lso_hdr_sz + sizeof (uint32_t) +
                    0xf) >> 4;  /* 0xf is for rounding up to a multiple of 16 */
        rsvd_lkey = (iov_attr->iov_flags & IBT_IOV_ALT_LKEY) ?
            iov_attr->iov_alt_lkey : state->hs_devlim.rsv_lkey;
        if ((iov_attr->iov_flags & IBT_IOV_NOSLEEP) == 0) {
                kmflag = KM_SLEEP;
                callback = DDI_DMA_SLEEP;
        } else {
                kmflag = KM_NOSLEEP;
                callback = DDI_DMA_DONTWAIT;
        }

        if (iov_attr->iov_flags & IBT_IOV_BUF) {
                mi_hdl = kmem_alloc(sizeof (*mi_hdl), kmflag);
                if (mi_hdl == NULL)
                        return (IBT_INSUFF_RESOURCE);
                sgl = wr->send.wr_sgl;
                _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sgl))

                status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
                    callback, NULL, &dmahdl);
                if (status != DDI_SUCCESS) {
                        kmem_free(mi_hdl, sizeof (*mi_hdl));
                        return (IBT_INSUFF_RESOURCE);
                }
                status = ddi_dma_buf_bind_handle(dmahdl, iov_attr->iov_buf,
                    DDI_DMA_RDWR | DDI_DMA_CONSISTENT, callback, NULL,
                    &dmacookie, &cookie_cnt);
                if (status != DDI_DMA_MAPPED) {
                        ddi_dma_free_handle(&dmahdl);
                        kmem_free(mi_hdl, sizeof (*mi_hdl));
                        return (ibc_get_ci_failure(0));
                }
                while (cookie_cnt-- > 0) {
                        if (nds > max_nds) {
                                status = ddi_dma_unbind_handle(dmahdl);
                                if (status != DDI_SUCCESS)
                                        HERMON_WARNING(state, "failed to "
                                            "unbind DMA mapping");
                                ddi_dma_free_handle(&dmahdl);
                                return (IBT_SGL_TOO_SMALL);
                        }
                        sgl[nds].ds_va = dmacookie.dmac_laddress;
                        sgl[nds].ds_key = rsvd_lkey;
                        sgl[nds].ds_len = (ib_msglen_t)dmacookie.dmac_size;
                        nds++;
                        if (cookie_cnt != 0)
                                ddi_dma_nextcookie(dmahdl, &dmacookie);
                }
                wr->send.wr_nds = nds;
                mi_hdl->imh_len = 1;
                mi_hdl->imh_dmahandle[0] = dmahdl;
                *mi_hdl_p = mi_hdl;
                return (IBT_SUCCESS);
        }

        if (iov_attr->iov_flags & IBT_IOV_RECV)
                sgl = wr->recv.wr_sgl;
        else
                sgl = wr->send.wr_sgl;
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*sgl))

        len = iov_attr->iov_list_len;
        for (i = 0, j = 0; j < len; j++) {
                if (iov_attr->iov[j].iov_len == 0)
                        continue;
                i++;
        }
        mi_hdl = kmem_alloc(sizeof (*mi_hdl) +
            (i - 1) * sizeof (ddi_dma_handle_t), kmflag);
        if (mi_hdl == NULL)
                return (IBT_INSUFF_RESOURCE);
        mi_hdl->imh_len = i;
        for (i = 0, j = 0; j < len; j++) {
                if (iov_attr->iov[j].iov_len == 0)
                        continue;
                status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr,
                    callback, NULL, &dmahdl);
                if (status != DDI_SUCCESS) {
                        ibt_status = IBT_INSUFF_RESOURCE;
                        goto fail2;
                }
                status = ddi_dma_addr_bind_handle(dmahdl, iov_attr->iov_as,
                    iov_attr->iov[j].iov_addr, iov_attr->iov[j].iov_len,
                    DDI_DMA_RDWR | DDI_DMA_CONSISTENT, callback, NULL,
                    &dmacookie, &cookie_cnt);
                if (status != DDI_DMA_MAPPED) {
                        ibt_status = ibc_get_ci_failure(0);
                        goto fail1;
                }
                if (nds + cookie_cnt > max_nds) {
                        ibt_status = IBT_SGL_TOO_SMALL;
                        goto fail2;
                }
                while (cookie_cnt-- > 0) {
                        sgl[nds].ds_va = dmacookie.dmac_laddress;
                        sgl[nds].ds_key = rsvd_lkey;
                        sgl[nds].ds_len = (ib_msglen_t)dmacookie.dmac_size;
                        nds++;
                        if (cookie_cnt != 0)
                                ddi_dma_nextcookie(dmahdl, &dmacookie);
                }
                mi_hdl->imh_dmahandle[i] = dmahdl;
                i++;
        }

        if (iov_attr->iov_flags & IBT_IOV_RECV)
                wr->recv.wr_nds = nds;
        else
                wr->send.wr_nds = nds;
        *mi_hdl_p = mi_hdl;
        return (IBT_SUCCESS);

fail1:
        ddi_dma_free_handle(&dmahdl);
fail2:
        while (--i >= 0) {
                status = ddi_dma_unbind_handle(mi_hdl->imh_dmahandle[i]);
                if (status != DDI_SUCCESS)
                        HERMON_WARNING(state, "failed to unbind DMA mapping");
                ddi_dma_free_handle(&mi_hdl->imh_dmahandle[i]);
        }
        kmem_free(mi_hdl, sizeof (*mi_hdl) +
            (len - 1) * sizeof (ddi_dma_handle_t));
        *mi_hdl_p = NULL;
        return (ibt_status);
}

/*
 * hermon_ci_unmap_mem_iov()
 * Unmap the memory
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_unmap_mem_iov(ibc_hca_hdl_t hca, ibc_mi_hdl_t mi_hdl)
{
        int             status, i;
        hermon_state_t  *state;

        state = (hermon_state_t *)hca;

        for (i = mi_hdl->imh_len; --i >= 0; ) {
                status = ddi_dma_unbind_handle(mi_hdl->imh_dmahandle[i]);
                if (status != DDI_SUCCESS)
                        HERMON_WARNING(state, "failed to unbind DMA mapping");
                ddi_dma_free_handle(&mi_hdl->imh_dmahandle[i]);
        }
        kmem_free(mi_hdl, sizeof (*mi_hdl) +
            (mi_hdl->imh_len - 1) * sizeof (ddi_dma_handle_t));
        return (IBT_SUCCESS);
}

/*
 * hermon_ci_alloc_lkey()
 * Allocate an empty memory region for use with FRWR.
 *    Context: Can be called from user or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_alloc_lkey(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
    ibt_lkey_flags_t flags, uint_t list_sz, ibc_mr_hdl_t *mr_p,
    ibt_pmr_desc_t *mem_desc_p)
{
        hermon_state_t          *state;
        hermon_pdhdl_t          pdhdl;
        hermon_mrhdl_t          mrhdl;
        int                     status;

        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mem_desc_p))

        ASSERT(mr_p != NULL);
        ASSERT(mem_desc_p != NULL);

        state = (hermon_state_t *)hca;
        pdhdl = (hermon_pdhdl_t)pd;

        if (!(state->hs_ibtfinfo.hca_attr->hca_flags2 & IBT_HCA2_MEM_MGT_EXT))
                return (IBT_NOT_SUPPORTED);

        status = hermon_mr_alloc_lkey(state, pdhdl, flags, list_sz, &mrhdl);
        if (status != DDI_SUCCESS) {
                return (status);
        }
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mrhdl))

        /* Fill in the mem_desc_p structure */
        mem_desc_p->pmd_iova = 0;
        mem_desc_p->pmd_phys_buf_list_sz = list_sz;
        mem_desc_p->pmd_lkey = mrhdl->mr_lkey;
        /* Only set RKey if remote access was requested */
        if (flags & IBT_KEY_REMOTE) {
                mem_desc_p->pmd_rkey = mrhdl->mr_rkey;
        }
        mem_desc_p->pmd_sync_required = B_FALSE;

        /* Return the Hermon MR handle */
        *mr_p = (ibc_mr_hdl_t)mrhdl;
        return (IBT_SUCCESS);
}

/* Physical Register Memory Region */
/*
 * hermon_ci_register_physical_mr()
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_physical_mr(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
    ibt_pmr_attr_t *mem_pattrs, void *ibtl_reserved, ibc_mr_hdl_t *mr_p,
    ibt_pmr_desc_t *mem_desc_p)
{
        return (IBT_NOT_SUPPORTED);
}

/*
 * hermon_ci_reregister_physical_mr()
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_reregister_physical_mr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr,
    ibc_pd_hdl_t pd, ibt_pmr_attr_t *mem_pattrs, void *ibtl_reserved,
    ibc_mr_hdl_t *mr_p, ibt_pmr_desc_t *mr_desc_p)
{
        return (IBT_NOT_SUPPORTED);
}

/* Mellanox FMR Support */
/*
 * hermon_ci_create_fmr_pool()
 * Creates a pool of memory regions suitable for FMR registration
 *    Context: Can be called from base context only
 */
static ibt_status_t
hermon_ci_create_fmr_pool(ibc_hca_hdl_t hca, ibc_pd_hdl_t pd,
    ibt_fmr_pool_attr_t *params, ibc_fmr_pool_hdl_t *fmr_pool_p)
{
        hermon_state_t  *state;
        hermon_pdhdl_t  pdhdl;
        hermon_fmrhdl_t fmrpoolhdl;
        int             status;

        state = (hermon_state_t *)hca;

        /* Check for valid PD handle pointer */
        if (pd == NULL) {
                return (IBT_PD_HDL_INVALID);
        }

        pdhdl = (hermon_pdhdl_t)pd;

        /*
         * Validate the access flags.  Both Remote Write and Remote Atomic
         * require the Local Write flag to be set
         */
        if (((params->fmr_flags & IBT_MR_ENABLE_REMOTE_WRITE) ||
            (params->fmr_flags & IBT_MR_ENABLE_REMOTE_ATOMIC)) &&
            !(params->fmr_flags & IBT_MR_ENABLE_LOCAL_WRITE)) {
                return (IBT_MR_ACCESS_REQ_INVALID);
        }

        status = hermon_create_fmr_pool(state, pdhdl, params, &fmrpoolhdl);
        if (status != DDI_SUCCESS) {
                return (status);
        }

        /* Set fmr_pool from hermon handle */
        *fmr_pool_p = (ibc_fmr_pool_hdl_t)fmrpoolhdl;

        return (IBT_SUCCESS);
}

/*
 * hermon_ci_destroy_fmr_pool()
 * Free all resources associated with an FMR pool.
 *    Context: Can be called from base context only.
 */
static ibt_status_t
hermon_ci_destroy_fmr_pool(ibc_hca_hdl_t hca, ibc_fmr_pool_hdl_t fmr_pool)
{
        hermon_state_t  *state;
        hermon_fmrhdl_t fmrpoolhdl;
        int             status;

        state = (hermon_state_t *)hca;
        fmrpoolhdl = (hermon_fmrhdl_t)fmr_pool;

        status = hermon_destroy_fmr_pool(state, fmrpoolhdl);
        return (status);
}

/*
 * hermon_ci_flush_fmr_pool()
 * Force a flush of the memory tables, cleaning up used FMR resources.
 *    Context: Can be called from interrupt or base context.
 */
static ibt_status_t
hermon_ci_flush_fmr_pool(ibc_hca_hdl_t hca, ibc_fmr_pool_hdl_t fmr_pool)
{
        hermon_state_t  *state;
        hermon_fmrhdl_t fmrpoolhdl;
        int             status;

        state = (hermon_state_t *)hca;

        fmrpoolhdl = (hermon_fmrhdl_t)fmr_pool;
        status = hermon_flush_fmr_pool(state, fmrpoolhdl);
        return (status);
}

/*
 * hermon_ci_register_physical_fmr()
 * From the 'pool' of FMR regions passed in, performs register physical
 * operation.
 *    Context: Can be called from interrupt or base context.
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_register_physical_fmr(ibc_hca_hdl_t hca,
    ibc_fmr_pool_hdl_t fmr_pool, ibt_pmr_attr_t *mem_pattr,
    void *ibtl_reserved, ibc_mr_hdl_t *mr_p, ibt_pmr_desc_t *mem_desc_p)
{
        hermon_state_t          *state;
        hermon_mrhdl_t          mrhdl;
        hermon_fmrhdl_t         fmrpoolhdl;
        int                     status;

        ASSERT(mem_pattr != NULL);
        ASSERT(mr_p != NULL);
        ASSERT(mem_desc_p != NULL);

        /* Grab the Hermon softstate pointer */
        state = (hermon_state_t *)hca;

        fmrpoolhdl = (hermon_fmrhdl_t)fmr_pool;

        status = hermon_register_physical_fmr(state, fmrpoolhdl, mem_pattr,
            &mrhdl, mem_desc_p);
        if (status != DDI_SUCCESS) {
                return (status);
        }

        /*
         * If region is mapped for streaming (i.e. noncoherent), then set
         * sync is required
         */
        _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(*mem_desc_p))
        mem_desc_p->pmd_sync_required = (mrhdl->mr_bindinfo.bi_flags &
            IBT_MR_NONCOHERENT) ? B_TRUE : B_FALSE;
        if (mem_desc_p->pmd_sync_required == B_TRUE) {
                /* Fill in DMA handle for future sync operations */
                _NOTE(NOW_INVISIBLE_TO_OTHER_THREADS(mrhdl->mr_bindinfo))
                mrhdl->mr_bindinfo.bi_dmahdl =
                    (ddi_dma_handle_t)mem_pattr->pmr_ma;
        }

        /* Return the Hermon MR handle */
        *mr_p = (ibc_mr_hdl_t)mrhdl;

        return (IBT_SUCCESS);
}

/*
 * hermon_ci_deregister_fmr()
 * Moves an FMR (specified by 'mr') to the deregistered state.
 *    Context: Can be called from base context only.
 */
static ibt_status_t
hermon_ci_deregister_fmr(ibc_hca_hdl_t hca, ibc_mr_hdl_t mr)
{
        hermon_state_t          *state;
        hermon_mrhdl_t          mrhdl;
        int                     status;

        /* Grab the Hermon softstate pointer */
        state = (hermon_state_t *)hca;
        mrhdl = (hermon_mrhdl_t)mr;

        /*
         * Deregister the memory region, either "unmap" the FMR or deregister
         * the normal memory region.
         */
        status = hermon_deregister_fmr(state, mrhdl);
        return (status);
}

static int
hermon_mem_alloc(hermon_state_t *state, size_t size, ibt_mr_flags_t flags,
    caddr_t *kaddrp, ibc_mem_alloc_hdl_t *mem_hdl)
{
        ddi_dma_handle_t        dma_hdl;
        ddi_dma_attr_t          dma_attr;
        ddi_acc_handle_t        acc_hdl;
        size_t                  real_len;
        int                     status;
        int                     (*ddi_cb)(caddr_t);
        ibc_mem_alloc_hdl_t     mem_alloc_hdl;

        hermon_dma_attr_init(state, &dma_attr);

        ddi_cb = (flags & IBT_MR_NOSLEEP) ? DDI_DMA_DONTWAIT : DDI_DMA_SLEEP;

        /* Allocate a DMA handle */
        status = ddi_dma_alloc_handle(state->hs_dip, &dma_attr, ddi_cb,
            NULL, &dma_hdl);
        if (status != DDI_SUCCESS) {
                return (DDI_FAILURE);
        }

        /* Allocate DMA memory */
        status = ddi_dma_mem_alloc(dma_hdl, size,
            &state->hs_reg_accattr, DDI_DMA_CONSISTENT, ddi_cb,
            NULL, kaddrp, &real_len, &acc_hdl);
        if (status != DDI_SUCCESS) {
                ddi_dma_free_handle(&dma_hdl);
                return (DDI_FAILURE);
        }

        /* Package the hermon_dma_info contents and return */
        mem_alloc_hdl = kmem_alloc(sizeof (**mem_hdl),
            (flags & IBT_MR_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP);
        if (mem_alloc_hdl == NULL) {
                ddi_dma_mem_free(&acc_hdl);
                ddi_dma_free_handle(&dma_hdl);
                return (DDI_FAILURE);
        }
        mem_alloc_hdl->ibc_dma_hdl = dma_hdl;
        mem_alloc_hdl->ibc_acc_hdl = acc_hdl;

        *mem_hdl = mem_alloc_hdl;

        return (DDI_SUCCESS);
}

/*
 * hermon_ci_alloc_io_mem()
 *      Allocate dma-able memory
 *
 */
static ibt_status_t
hermon_ci_alloc_io_mem(ibc_hca_hdl_t hca, size_t size, ibt_mr_flags_t mr_flag,
    caddr_t *kaddrp, ibc_mem_alloc_hdl_t *mem_alloc_hdl_p)
{
        hermon_state_t  *state;
        int             status;

        /* Grab the Hermon softstate pointer and mem handle */
        state = (hermon_state_t *)hca;

        /* Allocate the memory and handles */
        status = hermon_mem_alloc(state, size, mr_flag, kaddrp,
            mem_alloc_hdl_p);

        if (status != DDI_SUCCESS) {
                *mem_alloc_hdl_p = NULL;
                *kaddrp = NULL;
                return (status);
        }

        return (IBT_SUCCESS);
}


/*
 * hermon_ci_free_io_mem()
 * Unbind handl and free the memory
 */
/* ARGSUSED */
static ibt_status_t
hermon_ci_free_io_mem(ibc_hca_hdl_t hca, ibc_mem_alloc_hdl_t mem_alloc_hdl)
{
        /* Unbind the handles and free the memory */
        (void) ddi_dma_unbind_handle(mem_alloc_hdl->ibc_dma_hdl);
        ddi_dma_mem_free(&mem_alloc_hdl->ibc_acc_hdl);
        ddi_dma_free_handle(&mem_alloc_hdl->ibc_dma_hdl);
        kmem_free(mem_alloc_hdl, sizeof (*mem_alloc_hdl));

        return (IBT_SUCCESS);
}