/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */

/*
 * Copyright (c) 2010, Oracle and/or its affiliates. All rights reserved.
 */

/*
 * sol_uverbs.c
 *
 * Solaris OFED User Verbs kernel agent module
 *
 */
#include <sys/devops.h>
#include <sys/conf.h>
#include <sys/modctl.h>
#include <sys/types.h>
#include <sys/file.h>
#include <sys/errno.h>
#include <sys/open.h>
#include <sys/cred.h>
#include <sys/uio.h>
#include <sys/semaphore.h>
#include <sys/stat.h>
#include <sys/ddi.h>
#include <sys/sunddi.h>
#include <sys/ib/clients/of/ofa_solaris.h>

#include <sys/ib/ibtl/ibvti.h>
#include <sys/ib/clients/of/sol_ofs/sol_ofs_common.h>
#include <sys/ib/clients/of/ofed_kernel.h>
#include <sys/ib/clients/of/sol_uverbs/sol_uverbs.h>
#include <sys/ib/clients/of/sol_uverbs/sol_uverbs_event.h>
#include <sys/ib/clients/of/sol_uverbs/sol_uverbs_comp.h>
#include <sys/ib/clients/of/sol_uverbs/sol_uverbs_qp.h>

static void *statep;
static ibt_clnt_hdl_t   sol_uverbs_ib_clntp = NULL;

char    *sol_uverbs_dbg_str = "sol_uverbs";

static int sol_uverbs_attach(dev_info_t *dip, ddi_attach_cmd_t cmd);
static int sol_uverbs_detach(dev_info_t *dip, ddi_detach_cmd_t cmd);
static int sol_uverbs_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
    void **resultp);
static int sol_uverbs_prop_op(dev_t dev, dev_info_t *dip, ddi_prop_op_t prop_op,
    int flags, char *name, caddr_t valuep, int *lengthp);
static int sol_uverbs_open(dev_t *devp, int flag, int otyp, cred_t *cred);
static int sol_uverbs_close(dev_t dev, int flag, int otyp, cred_t *cred);
static int sol_uverbs_poll(dev_t, short, int, short *, struct pollhead **);
static int sol_uverbs_read(dev_t dev, struct uio *uiop, cred_t *credp);
static int sol_uverbs_mmap(dev_t dev, off_t sol_uverbs_mmap, int prot);
static int sol_uverbs_write(dev_t dev, struct uio *uiop, cred_t *credp);

static struct cb_ops sol_uverbs_cb_ops = {
        .cb_open        = sol_uverbs_open,
        .cb_close       = sol_uverbs_close,
        .cb_strategy    = nodev,
        .cb_print       = nodev,
        .cb_dump        = nodev,
        .cb_read        = sol_uverbs_read,
        .cb_write       = sol_uverbs_write,
        .cb_ioctl       = nodev,
        .cb_devmap      = nodev,
        .cb_mmap        = sol_uverbs_mmap,
        .cb_segmap      = nodev,
        .cb_chpoll      = sol_uverbs_poll,
        .cb_prop_op     = sol_uverbs_prop_op,
        .cb_str         = NULL,
        .cb_flag        = D_NEW | D_MP,
        .cb_rev         = CB_REV,
        .cb_aread       = nodev,
        .cb_awrite      = nodev
};

static struct dev_ops sol_uverbs_dev_ops = {
        .devo_rev       = DEVO_REV,
        .devo_refcnt    = 0,
        .devo_getinfo   = sol_uverbs_getinfo,
        .devo_identify  = nulldev,
        .devo_probe     = nulldev,
        .devo_attach    = sol_uverbs_attach,
        .devo_detach    = sol_uverbs_detach,
        .devo_reset     = nodev,
        .devo_cb_ops    = &sol_uverbs_cb_ops,
        .devo_bus_ops   = NULL,
        .devo_power     = nodev,
        .devo_quiesce   = ddi_quiesce_not_needed
};

static struct modldrv modldrv = {
        .drv_modops     = &mod_driverops,
        .drv_linkinfo   = "Solaris User Verbs driver",
        .drv_dev_ops    = &sol_uverbs_dev_ops
};

static struct modlinkage modlinkage = {
        .ml_rev                 = MODREV_1,
        .ml_linkage = {
                [0]             = &modldrv,
                [1]             = NULL,
        }
};

/*
 * User Object Tables for management of user resources. The tables are driver
 * wide, but each user context maintains a list of the objects it has created
 * that is used in cleanup.
 */
sol_ofs_uobj_table_t uverbs_uctxt_uo_tbl;
sol_ofs_uobj_table_t uverbs_upd_uo_tbl;
sol_ofs_uobj_table_t uverbs_uah_uo_tbl;
sol_ofs_uobj_table_t uverbs_umr_uo_tbl;
sol_ofs_uobj_table_t uverbs_ucq_uo_tbl;
sol_ofs_uobj_table_t uverbs_usrq_uo_tbl;
sol_ofs_uobj_table_t uverbs_uqp_uo_tbl;
sol_ofs_uobj_table_t uverbs_ufile_uo_tbl;

static void sol_uverbs_user_objects_init(void);
static void sol_uverbs_user_objects_fini(void);

/*
 * Open Fabric User Verbs API, command table. See ib_user_verbs.h for
 * definitions.
 */
static int (*uverbs_cmd_table[])(uverbs_uctxt_uobj_t *uctxt, char *buf,
        int in_len, int out_len) = {

        [IB_USER_VERBS_CMD_GET_CONTEXT]         = sol_uverbs_get_context,
        [IB_USER_VERBS_CMD_QUERY_DEVICE]        = sol_uverbs_query_device,
        [IB_USER_VERBS_CMD_QUERY_PORT]          = sol_uverbs_query_port,
        [IB_USER_VERBS_CMD_ALLOC_PD]            = sol_uverbs_alloc_pd,
        [IB_USER_VERBS_CMD_DEALLOC_PD]          = sol_uverbs_dealloc_pd,
        [IB_USER_VERBS_CMD_REG_MR]              = sol_uverbs_reg_mr,
        [IB_USER_VERBS_CMD_DEREG_MR]            = sol_uverbs_dereg_mr,
        [IB_USER_VERBS_CMD_CREATE_COMP_CHANNEL] =
                                        sol_uverbs_create_comp_channel,
        [IB_USER_VERBS_CMD_CREATE_CQ]           = sol_uverbs_create_cq,
        [IB_USER_VERBS_CMD_RESIZE_CQ]           = sol_uverbs_resize_cq,
        [IB_USER_VERBS_CMD_POLL_CQ]             = sol_uverbs_poll_cq,
        [IB_USER_VERBS_CMD_REQ_NOTIFY_CQ]       = sol_uverbs_req_notify_cq,
        [IB_USER_VERBS_CMD_DESTROY_CQ]          = sol_uverbs_destroy_cq,
        [IB_USER_VERBS_CMD_CREATE_QP]           = sol_uverbs_create_qp,
        [IB_USER_VERBS_CMD_QUERY_QP]            = sol_uverbs_query_qp,
        [IB_USER_VERBS_CMD_MODIFY_QP]           = sol_uverbs_modify_qp,
        [IB_USER_VERBS_CMD_DESTROY_QP]          = sol_uverbs_destroy_qp,
        [IB_USER_VERBS_CMD_POST_SEND]           = sol_uverbs_dummy_command,
        [IB_USER_VERBS_CMD_POST_RECV]           = sol_uverbs_dummy_command,
        [IB_USER_VERBS_CMD_POST_SRQ_RECV]       = sol_uverbs_dummy_command,
        [IB_USER_VERBS_CMD_CREATE_AH]           = sol_uverbs_create_ah,
        [IB_USER_VERBS_CMD_DESTROY_AH]          = sol_uverbs_destroy_ah,
        [IB_USER_VERBS_CMD_ATTACH_MCAST]        = sol_uverbs_attach_mcast,
        [IB_USER_VERBS_CMD_DETACH_MCAST]        = sol_uverbs_detach_mcast,
        [IB_USER_VERBS_CMD_CREATE_SRQ]          = sol_uverbs_create_srq,
        [IB_USER_VERBS_CMD_MODIFY_SRQ]          = sol_uverbs_modify_srq,
        [IB_USER_VERBS_CMD_QUERY_SRQ]           = sol_uverbs_query_srq,
        [IB_USER_VERBS_CMD_DESTROY_SRQ]         = sol_uverbs_destroy_srq,

                /* TODO - XRC */

        [IB_USER_VERBS_CMD_CREATE_XRC_SRQ]      = sol_uverbs_dummy_command,
        [IB_USER_VERBS_CMD_OPEN_XRC_DOMAIN]     = sol_uverbs_dummy_command,
        [IB_USER_VERBS_CMD_CLOSE_XRC_DOMAIN]    = sol_uverbs_dummy_command,
        [IB_USER_VERBS_CMD_CREATE_XRC_RCV_QP]   = sol_uverbs_dummy_command,
        [IB_USER_VERBS_CMD_MODIFY_XRC_RCV_QP]   = sol_uverbs_dummy_command,
        [IB_USER_VERBS_CMD_QUERY_XRC_RCV_QP]    = sol_uverbs_dummy_command,
        [IB_USER_VERBS_CMD_REG_XRC_RCV_QP]      = sol_uverbs_dummy_command,
        [IB_USER_VERBS_CMD_UNREG_XRC_RCV_QP]    = sol_uverbs_dummy_command,
        [IB_USER_VERBS_CMD_QUERY_GID]           = sol_uverbs_query_gid,
        [IB_USER_VERBS_CMD_QUERY_PKEY]          = sol_uverbs_query_pkey,
};

/*
 * Function:
 *      sol_uverbs_hca_open
 * Input:
 *      mod_ctxt        - Pointer to the user verbs module context.
 * Output:
 *      None
 * Returns:
 *      Zero on success, else error code.
 * Description:
 *      Register as a client with the IBT framework and open all of the
 *      HCA's present.
 */
static int
sol_uverbs_hca_open(uverbs_module_context_t *mod_ctxt)
{
        int                     status;
        int                     hca_ndx;
#ifdef DEBUG
        llist_head_t            *entry;
        sol_uverbs_hca_t        *temp;
#endif

        mod_ctxt->hca_count     = 0;
        mod_ctxt->hca_guid_list = NULL;
        mod_ctxt->hcas          = NULL;
        mod_ctxt->clnt_hdl      = NULL;

        mod_ctxt->clnt_modinfo.mi_ibt_version   = IBTI_V_CURR;
        mod_ctxt->clnt_modinfo.mi_clnt_class    = IBT_USER;
        mod_ctxt->clnt_modinfo.mi_async_handler = uverbs_async_event_handler;
        mod_ctxt->clnt_modinfo.mi_reserved      = NULL;
        mod_ctxt->clnt_modinfo.mi_clnt_name     = "sol_uverbs";

        status = ibt_attach(&mod_ctxt->clnt_modinfo, mod_ctxt->dip,
            mod_ctxt, &mod_ctxt->clnt_hdl);

        if (status != IBT_SUCCESS) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "hca_open:ibt_attach fail %d", status);
                status = ENODEV;
                goto out_err;
        }
        if (sol_uverbs_ib_clntp == NULL)
                sol_uverbs_ib_clntp  = mod_ctxt->clnt_hdl;

        mod_ctxt->hca_count = ibt_get_hca_list(&mod_ctxt->hca_guid_list);

        if (mod_ctxt->hca_count == 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "hca_open: Zero HCAs on this system!");
                status = ENODEV;
                goto out_err;
        }

        if (mod_ctxt->hca_count > SOL_UVERBS_DRIVER_MAX_HCA_MINOR) {
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "hca_open: HCA count %d exceeds max %d",
                    mod_ctxt->hca_count, SOL_UVERBS_DRIVER_MAX_HCA_MINOR);
                status =  ENODEV;
                goto out_err;
        }

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "hca_open: No. of HCAs present %d", mod_ctxt->hca_count);

        mod_ctxt->hcas = kmem_zalloc(mod_ctxt->hca_count *
            sizeof (sol_uverbs_hca_t), KM_SLEEP);
        ASSERT(mod_ctxt->hcas != NULL);

        /*
         * Note: we open these in the reverse order of the guid list, although
         * this is technically not required it is done this way so that the
         * mapping will be in same order as the interfaces. Also note, that we
         * provide a guid property, and the guid should be used to map a verbs
         * device to an interface (i.e. don't depend on the order).
         */
        for (hca_ndx = 0; hca_ndx < mod_ctxt->hca_count; hca_ndx++) {
                status = ibt_open_hca(mod_ctxt->clnt_hdl,
                    mod_ctxt->hca_guid_list[mod_ctxt->hca_count - hca_ndx -1],
                    &mod_ctxt->hcas[hca_ndx].hdl);
                if (status != IBT_SUCCESS) {
                        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                            "hca_open: ibt_open_hca() returned %d",
                            status);
                        goto out_err;
                }

                mod_ctxt->hcas[hca_ndx].clnt_hdl = mod_ctxt->clnt_hdl;
                llist_head_init(&mod_ctxt->hcas[hca_ndx].list,
                    &mod_ctxt->hcas[hca_ndx]);
                mutex_init(&mod_ctxt->hcas[hca_ndx].event_handler_lock, NULL,
                    MUTEX_DRIVER, NULL);
                llist_head_init(&mod_ctxt->hcas[hca_ndx].event_handler_list,
                    NULL);
                mutex_init(&mod_ctxt->hcas[hca_ndx].client_data_lock, NULL,
                    MUTEX_DRIVER, NULL);
                llist_head_init(&mod_ctxt->hcas[hca_ndx].client_data_list,
                    NULL);

                mutex_enter(&sol_uverbs_hca_lock);
                llist_add_tail(&mod_ctxt->hcas[hca_ndx].list,
                    &sol_uverbs_hca_list);
                mutex_exit(&sol_uverbs_hca_lock);

                mod_ctxt->hcas[hca_ndx].guid =
                    mod_ctxt->hca_guid_list[mod_ctxt->hca_count - hca_ndx -1];

                /*
                 * Get a cached copy of the HCA's attributes for easy access.
                 */
                status = ibt_query_hca(mod_ctxt->hcas[hca_ndx].hdl,
                    &mod_ctxt->hcas[hca_ndx].attr);
                if (status != IBT_SUCCESS) {
                        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                            "hca_open: ibt_query_hca() failed "
                            "(status=%d)", status);
                        goto out_err;
                }

                /* Note : GUID is in host order here */
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "hca_open: HCA index %d, HCA GUID: 0x%016llX",
                    hca_ndx, (u_longlong_t)mod_ctxt->hcas[hca_ndx].guid);
        }

#ifdef DEBUG
        mutex_enter(&sol_uverbs_hca_lock);
        list_for_each(entry, &sol_uverbs_hca_list) {
                temp = (sol_uverbs_hca_t *)entry->ptr;
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "HCA list: entry: %p, handle: %p, "
                    "GUID: 0x%016llX", (void *)entry, (void *)temp->hdl,
                    (u_longlong_t)temp->guid);
        }
        mutex_exit(&sol_uverbs_hca_lock);
#endif
        return (0);

out_err:
        /*
         * Note, cleanup of hca list and associated resources is done via
         * uverbs_hca_close called outside this routine in the case of bad
         * status.
         */
        return (status);
}

/*
 * Function:
 *      sol_uverbs_hca_close
 * Input:
 *      mod_ctxt        - Pointer to the module context.
 * Output:
 *      None
 * Returns:
 *      None
 * Description:
 *      Close all of the IBT HCAs opened by the driver and detach from
 *      the IBT framework.
 */
static void
sol_uverbs_hca_close(uverbs_module_context_t *mod_ctxt)
{
        int    hca_ndx;

        if (mod_ctxt->hcas != NULL) {
                mutex_enter(&sol_uverbs_hca_lock);
                llist_head_init(&sol_uverbs_hca_list, NULL);
                mutex_exit(&sol_uverbs_hca_lock);
                for (hca_ndx = 0; hca_ndx < mod_ctxt->hca_count; hca_ndx++) {
                        if (mod_ctxt->hcas[hca_ndx].hdl != NULL) {
                                mutex_destroy(&mod_ctxt->hcas[hca_ndx].
                                    event_handler_lock);
                                mutex_destroy(&mod_ctxt->hcas[hca_ndx].
                                    client_data_lock);
                                (void) ibt_close_hca(mod_ctxt->
                                    hcas[hca_ndx].hdl);
                        }
                }
                kmem_free(mod_ctxt->hcas,
                    mod_ctxt->hca_count * sizeof (sol_uverbs_hca_t));
                mod_ctxt->hcas = NULL;
        }

        if ((mod_ctxt->hca_guid_list != NULL) && (mod_ctxt->hca_count > 0)) {
                ibt_free_hca_list(mod_ctxt->hca_guid_list, mod_ctxt->hca_count);
                mod_ctxt->hca_count     = 0;
                mod_ctxt->hca_guid_list = NULL;
        }

        if (mod_ctxt->clnt_hdl != NULL) {
                (void) ibt_detach(mod_ctxt->clnt_hdl);
                mod_ctxt->clnt_hdl = NULL;
        }
}

/*
 * Function:
 *      _init
 * Input:
 *      None
 * Output:
 *      None
 * Returns:
 *      DDI_SUCCESS  on success, else error code.
 * Description:
 *      Perform Solaris OFED user verbs kernel agent driver initialization.
 */
int
_init(void)
{
        int     error;

        error = ddi_soft_state_init(&statep,
            sizeof (uverbs_module_context_t), 0);

        if (error != 0) {
                return (error);
        }

        sol_uverbs_user_objects_init();
        if (sol_uverbs_common_hca_init()) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_hca_init() failed");
                ddi_soft_state_fini(&statep);
                return (ENODEV);
        }

        error = mod_install(&modlinkage);
        if (error != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs: mod_install failed!!");
                sol_uverbs_common_hca_fini();
                sol_uverbs_user_objects_fini();
                ddi_soft_state_fini(&statep);
        }
        return (error);
}

/*
 * Function:
 *      _info
 * Input:
 *      modinfop        - Pointer to an opqque modinfo structure.
 * Output:
 *      modinfop        - Updated structure.
 * Returns:
 *      The mod_info() return code.
 * Description:
 *      Return information about the loadable module via the mod_info()
 *      kernel function call.
 */
int
_info(struct modinfo *modinfop)
{
        return (mod_info(&modlinkage, modinfop));
}

/*
 * Function:
 *      _fini
 * Input:
 *      None
 * Output:
 *      None
 * Returns:
 *      DDI_SUCCESS  on success, else error code returned by
 *      mod_remove kernel function.
 * Description:
 *      Perform Solaris OFED user verbs kernel agent driver cleanup.
 */
int
_fini(void)
{
        int    rc;

        rc = mod_remove(&modlinkage);
        if (!rc) {
                sol_uverbs_common_hca_fini();
                sol_uverbs_user_objects_fini();
        }
        return (rc);
}

int
sol_uverbs_attach(dev_info_t *dip, ddi_attach_cmd_t cmd)
{
        uverbs_module_context_t *mod_ctxt;
        int                     rc, instance, hca_ndx;

        switch (cmd) {
                case DDI_ATTACH:
                        break;
                case DDI_RESUME:
                        return (DDI_SUCCESS);
                default:
                        return (DDI_FAILURE);
        }

        /*
         * Allocate a soft data structure based on this dev info
         */
        instance = ddi_get_instance(dip);
        if (instance != 0) {
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "attach: bad instance number %d", instance);
                return (DDI_FAILURE);
        }

        if (ddi_soft_state_zalloc(statep, instance) != DDI_SUCCESS) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "attach: bad state zalloc");
                return (DDI_FAILURE);
        }

        mod_ctxt = ddi_get_soft_state(statep, instance);
        if (mod_ctxt == NULL) {
                ddi_soft_state_free(statep, instance);
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "attach: cannot get soft state");
                return (DDI_FAILURE);
        }

        /*
         * Save off our private context in the dev_info
         */
        mod_ctxt->dip = dip;
        ddi_set_driver_private(dip, mod_ctxt);

        /*
         * Opening of the hca will perform the ibt_attach and build a list of
         * devices.
         */
        rc = sol_uverbs_hca_open(mod_ctxt);
        if (rc) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "attach: sol_uverbs_hca_open() (rc=%d)", rc);
                goto error;
        }

        /*
         * Export our ABI revision as a property.
         */
        rc = ddi_prop_update_int(makedevice(ddi_driver_major(dip), 0),
            dip, "abi-version", IB_USER_VERBS_ABI_VERSION);
        if (rc != DDI_SUCCESS) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "attach: could not add abi-version property");
        }

        /*
         * Create the filesystem device node for each HCA.
         */
        for (hca_ndx = 0; hca_ndx < mod_ctxt->hca_count; hca_ndx++) {
                char name[20];

                (void) snprintf(name, 20, "uverbs%d", hca_ndx);
                rc = ddi_create_minor_node(dip, name, S_IFCHR, hca_ndx,
                    DDI_PSEUDO, 0);
                if (rc != DDI_SUCCESS) {
                        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                            "attach: could not add character node");
                        goto error;
                }

                rc = ddi_prop_update_int64(makedevice(ddi_driver_major(dip),
                    hca_ndx), dip, "guid", mod_ctxt->hcas[hca_ndx].guid);
                if (rc != DDI_SUCCESS) {
                        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                            "attach: could not add GUID property");
                }

                rc = ddi_prop_update_int(makedevice(ddi_driver_major(dip),
                    hca_ndx), dip, "vendor-id",
                    mod_ctxt->hcas[hca_ndx].attr.hca_vendor_id);
                if (rc != DDI_SUCCESS) {
                        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                            "attach: could not add vendor-id property");
                }

                rc = ddi_prop_update_int(makedevice(ddi_driver_major(dip),
                    hca_ndx), dip, "device-id", mod_ctxt->
                    hcas[hca_ndx].attr.hca_device_id);
                if (rc != DDI_SUCCESS) {
                        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                            "attach: could not add device-id property");
                }
        }

        rc = ddi_create_minor_node(dip, "ucma",  S_IFCHR,
            SOL_UVERBS_DRIVER_MAX_HCA_MINOR, DDI_PSEUDO, 0);

        if (rc != DDI_SUCCESS) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "attach: could not add minor for ucma");
                goto error;
        }

        rc = ddi_create_minor_node(dip, "event",  S_IFCHR,
            SOL_UVERBS_DRIVER_EVENT_MINOR, DDI_PSEUDO, 0);

        if (rc != DDI_SUCCESS) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "attach: could not add minor for events");
                goto error;
        }

        ddi_report_dev(dip);

        return (DDI_SUCCESS);

error:
        /*
         * Cleanup any resources and dettach.
         */
        sol_uverbs_hca_close(mod_ctxt);
        ddi_soft_state_free(statep, instance);

        return (rc);
}

/*
 * Function:
 *      sol_uverbs_detach
 * Input:
 *      dip     - A pointer to the devices dev_info_t structure.
 *      cmd     - Type of detach (DDI_DETACH or DDI_SUSPEND).
 * Output:
 *      None
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      Detaches thea driver module and will cause the driver to close
 *      the underlying IBT HCA and detach from the IBT driver.  Note
 *      that this call will fail if user verb consumers or ucma have a
 *      sol_uverbs device open.
 */
static int
sol_uverbs_detach(dev_info_t *dip, ddi_detach_cmd_t cmd)
{
        int                     instance;
        uverbs_module_context_t *mod_ctxt;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "detach()");

        if (cmd != DDI_DETACH) {
                return (DDI_FAILURE);
        }

        instance = ddi_get_instance(dip);
        if (instance != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "detach: bad instance number 0x%x", instance);
                return (DDI_FAILURE);
        }

        rw_enter(&uverbs_uctxt_uo_tbl.uobj_tbl_lock, RW_WRITER);
        if (uverbs_uctxt_uo_tbl.uobj_tbl_uo_cnt > 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "detach(): device in use");
                rw_exit(&uverbs_uctxt_uo_tbl.uobj_tbl_lock);
                return (DDI_FAILURE);
        }
        rw_exit(&uverbs_uctxt_uo_tbl.uobj_tbl_lock);

        mod_ctxt = ddi_get_soft_state(statep, instance);

        /*
         * Sanity check, do not detach if other kernel agents
         * are still using sol_uverbs IBT handles.
         */
        mutex_enter(&sol_uverbs_hca_lock);
        if (!llist_empty(&sol_uverbs_client_list)) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "detach: agents still registered");
                mutex_exit(&sol_uverbs_hca_lock);
                return (DDI_FAILURE);
        }
        mutex_exit(&sol_uverbs_hca_lock);

        /*
         * Hca close will perform the detach from IBTF.
         */
        sol_uverbs_hca_close(mod_ctxt);

        ddi_soft_state_free(statep, instance);
        ddi_remove_minor_node(dip, NULL);
        return (DDI_SUCCESS);
}

/*
 * Function:
 *      sol_uverbs_getinfo
 * Input:
 *      dip     - Deprecated, do not use.
 *      cmd     - Command argument (DDI_INFO_DEVT2DEVINFO or
 *                DDI_INFO_DEVT2INSTANCE).
 *      arg     - Command specific argument.
 *      resultp - Pointer to place results.
 * Output:
 *      resultp - Location is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      Depending on the request (cmd) return either the dev_info_t pointer
 *      associated with the dev_info_t specified, or the instance.  Note
 *      that we have only a single instance.
 */
/* ARGSUSED */
static int
sol_uverbs_getinfo(dev_info_t *dip, ddi_info_cmd_t cmd, void *arg,
    void **resultp)
{
        uverbs_module_context_t *mod_ctxt;

        switch (cmd) {
                case DDI_INFO_DEVT2DEVINFO:
                        mod_ctxt = ddi_get_soft_state(statep, 0);
                        if (!mod_ctxt) {
                                return (DDI_FAILURE);
                        }
                        *resultp = (void *)mod_ctxt->dip;
                        return (DDI_SUCCESS);

                case DDI_INFO_DEVT2INSTANCE:
                        *resultp = 0;
                        return (DDI_SUCCESS);

                default:
                        return (DDI_FAILURE);
        }
}

/*
 * Function:
 *      sol_uverbs_prop_op
 * Input:
 *      dev     - The device number associated with this device.
 *      dip     - A pointer to the device information structure for this device.
 *      prop_op - Property operator (PROP_LEN, PROP_LEN_AND_VAL_BUF, or
 *                PROP_LEN_AND_VAL_ALLOC).
 *      flags   - Only possible flag value is DDI_PROP_DONTPASS.
 *      name    - Pointer to the property to be interrogated.
 *      valuep  - Address of pointer if ALLOC, otherwise a pointer to the
 *                users buffer.
 *      lengthp - Pointer to update with property length.
 * Output:
 *      valuep  - Updated with the property value.
 *      lenghtp - Updated with the property length.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      Driver entry point to report the values of certain properties of the
 *      driver or  device.
 */
static int
sol_uverbs_prop_op(dev_t dev, dev_info_t *dip, ddi_prop_op_t prop_op, int flags,
    char *name, caddr_t valuep, int *lengthp)
{
        return (ddi_prop_op(dev, dip, prop_op, flags, name, valuep, lengthp));

}

static uverbs_uctxt_uobj_t *sol_uverbs_alloc_uctxt(dev_t *,
    uverbs_module_context_t *, minor_t);

/*
 * Function:
 *      sol_uverbs_open
 * Input:
 *      devp    - A pointer to the device number.
 *      flag    - Flags specified by caller (FEXCL, FNDELAY, FREAD, FWRITE).
 *      otyp    - Open type (OTYP_BLK, OTYP_CHR, OTYP_LYR).
 *      cred    - Pointer to the callers credentials.
 * Output:
 *      devp    - On success devp has been cloned to point to a unique minor
 *                device.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      Handles a user process open of a specific user verbs minor device by
 *      allocating a user context user object and creating a unique device
 *      to identify the user.  Note: The first SOL_UVERBS_DRIVER_MAX_MINOR
 *      minor numbers are reserved for :
 *              0 to SOL_UVERBS_DRIVER_MAX_HCA_MINOR - 1 : actual HCA devices
 *              SOL_UVERBS_DRIVER_MAX_HCA_MINOR          : UCMA node
 *              SOL_UVERBS_DRIVER_EVENT_MINOR            :
 *                      Event file for opening an event file for completion
 *                      or async notifications.
 */
/* ARGSUSED */
static int
sol_uverbs_open(dev_t *devp, int flag, int otyp, cred_t *cred)
{
        uverbs_module_context_t *mod_ctxt;
        uverbs_uctxt_uobj_t     *uctxt;
        int                     minor;

        /* Char only */
        if (otyp != OTYP_CHR) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "open: not CHR");
                return (EINVAL);
        }

        mod_ctxt = ddi_get_soft_state(statep, 0);
        if (mod_ctxt == NULL) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "open: get soft state failed");
                return (ENXIO);
        }

        minor = getminor(*devp);

        /*
         * Special case of ucma module.
         */
        if (minor == SOL_UVERBS_DRIVER_MAX_HCA_MINOR) {
                extern cred_t   *kcred;

                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "open: ucma_open");
                if (cred != kcred) {
                        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                            "open: ucma_open non-kernel context");
                        return (ENOTSUP);
                }

                return (DDI_SUCCESS);
        }

        /*
         * If this is not an open for sol_uverbs event file,
         * A device minor number must be less than the user verb max
         * minor device number and the HCA count.
         */
        if (minor != SOL_UVERBS_DRIVER_EVENT_MINOR &&
            (minor >= SOL_UVERBS_DRIVER_MAX_HCA_MINOR ||
            minor >= mod_ctxt->hca_count)) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "open: bad minor %d", minor);
                return (ENODEV);
        }
        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "open() - minor %d", minor);

        /*
         * Allocate a user context and return a unique ID that can be used
         * in identify the new user context object.  Create a clone device
         * that uses this unique ID as the minor number.  Allocation of the
         * user context object places one reference against it; which will
         * be held until the device is closed.
         *
         * sol_uverbs_alloc_uctxt() returns a sucessful allocation of uctx
         * with the uobj uo_lock held for WRITTER.
         */
        uctxt = sol_uverbs_alloc_uctxt(devp, mod_ctxt, minor);
        if (!uctxt)  {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "open: user context alloc failed");
                return (ENODEV);
        }

        /*
         * Indicate the object is alive and release the user object write lock
         * which was placed on the user context at allocation.
         */
        uctxt->uobj.uo_live = 1;
        rw_exit(&uctxt->uobj.uo_lock);

        return (DDI_SUCCESS);
}

/*
 * Function:
 *      sol_uverbs_close
 * Input:
 *      dev     - Device number.
 *      flag    - File status flag.
 *      otyp    - Open type.
 *      cred    - A pointer to the callers credientials.
 * Output:
 *      None
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      Handles a user process close of a specific user verbs minor device by
 *      freeing any user objects this process may still have allocated and
 *      deleting the associated user context object.
 */
/* ARGSUSED */
static int
sol_uverbs_close(dev_t dev, int flag, int otyp, cred_t *cred)
{
        minor_t                 id = getminor(dev);
        genlist_entry_t         *entry, *new_entry;
        uverbs_uctxt_uobj_t     *uctxt;
        int                     rc;
        genlist_t               tmp_genlist;

        /*
         * HCA specific device nodes created during attach are been
         * closed. Return SUCCESS.
         */
        if (id < SOL_UVERBS_DRIVER_MAX_MINOR) {
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "uverbs_close: dev_t %x, minor %x < %x",
                    dev, id, SOL_UVERBS_DRIVER_MAX_MINOR);
                return (0);
        }

        /*
         * Must be a user or kernel open, i.e. not a minor node that
         * that represents a user verbs device.  If it is the UCMA
         * nothing needs to be done.
         */
        if (id == SOL_UVERBS_DRIVER_MAX_HCA_MINOR) {
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "uverbs_close: ucma close");
                return (DDI_SUCCESS);
        }

        uctxt = uverbs_uobj_get_uctxt_write(id - SOL_UVERBS_DRIVER_MAX_MINOR);
        if (uctxt == NULL) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_close: Unknown user context");
                return (ENXIO);
        }
        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "uverbs_close- "
            "uctxt %p", uctxt);

        /*
         * Remove from the user context resource table, cleanup all
         * user resources that may still be hanging around.
         */
        if (!sol_ofs_uobj_remove(&uverbs_uctxt_uo_tbl, &uctxt->uobj)) {
                /*
                 * It was already removed, drop the lock held from
                 * get above and exit.
                 */
                sol_ofs_uobj_put(&uctxt->uobj);
                return (ENXIO);
        }

        if (uctxt->uctxt_type == SOL_UVERBS_UCTXT_ASYNC ||
            uctxt->uctxt_type == SOL_UVERBS_UCTXT_COMPL) {
                uverbs_uctxt_uobj_t     *verbs_uctxt;

                SOL_OFS_DPRINTF_L4(sol_uverbs_dbg_str,
                    "uverbs_close: Async or Compl user context");

                /*
                 * Verbs uctxt has already been freed, just return.
                 */
                if (!uctxt->uctxt_verbs_id) {
                        sol_ofs_uobj_put(&uctxt->uobj);
                        sol_ofs_uobj_deref(&uctxt->uobj, sol_ofs_uobj_free);
                        return (0);
                }

                /*
                 * Verbs uctxt has not been freed. Close the ufile. This
                 * also frees the ufile if reference count is 0.
                 */
                verbs_uctxt = uverbs_uobj_get_uctxt_write(
                    uctxt->uctxt_verbs_id - SOL_UVERBS_DRIVER_MAX_MINOR);

                if (verbs_uctxt &&
                    uctxt->uctxt_type == SOL_UVERBS_UCTXT_ASYNC) {
                        sol_uverbs_event_file_close(verbs_uctxt->async_evfile);
                        verbs_uctxt->async_evfile = NULL;
                } else if (uctxt->comp_evfile) {
                        uctxt->comp_evfile = NULL;
                }
                if (verbs_uctxt)
                        sol_ofs_uobj_put(&verbs_uctxt->uobj);

                sol_ofs_uobj_put(&uctxt->uobj);
                sol_ofs_uobj_deref(&uctxt->uobj, sol_ofs_uobj_free);
                return (0);
        } else if (uctxt->uctxt_type == SOL_UVERBS_UCTXT_EVENT) {
                sol_ofs_uobj_put(&uctxt->uobj);
                sol_ofs_uobj_deref(&uctxt->uobj, sol_ofs_uobj_free);
                return (0);
        }

        ASSERT(uctxt->hca != NULL);

        /*
         * Release resources that may still be held by this user context.
         * Remove the resources from the associated resource managment
         * table and free it.
         */
        mutex_enter(&uctxt->lock);

        entry = remove_genlist_head(&uctxt->ah_list);
        while (entry) {
                uverbs_uah_uobj_t *uah = (uverbs_uah_uobj_t *)entry->data;

                rw_enter(&(uah->uobj.uo_lock), RW_WRITER);
                (void) sol_ofs_uobj_remove(&uverbs_uah_uo_tbl, &uah->uobj);
                rw_exit(&(uah->uobj.uo_lock));
                (void) ibt_free_ah(uctxt->hca->hdl, uah->ah);
                sol_ofs_uobj_free(&uah->uobj);

                kmem_free((void *)entry, sizeof (genlist_entry_t));
                entry = remove_genlist_head(&uctxt->ah_list);
        }

        init_genlist(&tmp_genlist);
        entry = remove_genlist_head(&uctxt->qp_list);
        while (entry) {
                uverbs_uqp_uobj_t *uqp = (uverbs_uqp_uobj_t *)entry->data;

                /* Free unreaped asynchronous events.  */
                uverbs_release_uqp_uevents(uctxt->async_evfile, uqp);

                /*
                 * If ucma has disabled QP free for this QP, set the
                 * uqp_free_state to FREE_PENDING. Free QP if not.
                 */
                rw_enter(&(uqp->uobj.uo_lock), RW_WRITER);
                if (uqp->uqp_free_state != SOL_UVERBS2UCMA_ENABLE_QP_FREE) {
                        new_entry = add_genlist(&tmp_genlist, entry->data,
                            entry->data_context);
                        uqp->list_entry = new_entry;
                        uqp->uqp_free_state = SOL_UVERBS2UCMA_FREE_PENDING;
                        rw_exit(&(uqp->uobj.uo_lock));
                } else {
                        uqp->list_entry = NULL;
                        mutex_exit(&uctxt->lock);
                        sol_ofs_uobj_ref(&uqp->uobj);
                        rc = uverbs_uqp_free(uqp, uctxt);
                        mutex_enter(&uctxt->lock);
                        if (rc)
                                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                                    "uqp_free(%p) failed", uqp);
                }
                kmem_free(entry, sizeof (genlist_entry_t));
                entry = remove_genlist_head(&uctxt->qp_list);
        }
        (uctxt->qp_list).count = tmp_genlist.count;
        (uctxt->qp_list).head = tmp_genlist.head;
        (uctxt->qp_list).tail = tmp_genlist.tail;

        init_genlist(&tmp_genlist);
        entry = remove_genlist_head(&uctxt->cq_list);
        while (entry) {
                uverbs_ucq_uobj_t *ucq = (uverbs_ucq_uobj_t *)entry->data;

                rw_enter(&(ucq->uobj.uo_lock), RW_WRITER);

                /* Free events associated with the CQ.  */
                uverbs_release_ucq_channel(uctxt, ucq->comp_chan, ucq);

                if (ucq->active_qp_cnt) {
                        new_entry = add_genlist(&tmp_genlist, entry->data,
                            entry->data_context);
                        ucq->list_entry = new_entry;
                        ucq->free_pending = 1;
                        rw_exit(&(ucq->uobj.uo_lock));
                } else {
                        ucq->list_entry = NULL;
                        sol_ofs_uobj_ref(&ucq->uobj);
                        mutex_exit(&uctxt->lock);
                        rc = uverbs_ucq_free(ucq, uctxt);
                        mutex_enter(&uctxt->lock);
                        if (rc)
                                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                                    "ucq_free(%p) failed", ucq);
                }

                kmem_free((void *)entry, sizeof (genlist_entry_t));
                entry = remove_genlist_head(&uctxt->cq_list);
        }
        (uctxt->cq_list).count = tmp_genlist.count;
        (uctxt->cq_list).head = tmp_genlist.head;
        (uctxt->cq_list).tail = tmp_genlist.tail;

        init_genlist(&tmp_genlist);
        entry = remove_genlist_head(&uctxt->srq_list);
        while (entry) {
                uverbs_usrq_uobj_t *usrq = (uverbs_usrq_uobj_t *)entry->data;

                rw_enter(&(usrq->uobj.uo_lock), RW_WRITER);

                /* Free unreaped asynchronous events.  */
                uverbs_release_usrq_uevents(uctxt->async_evfile, usrq);

                if (usrq->active_qp_cnt) {
                        new_entry = add_genlist(&tmp_genlist, entry->data,
                            entry->data_context);
                        usrq->list_entry = new_entry;
                        usrq->free_pending = 1;
                        rw_exit(&(usrq->uobj.uo_lock));
                } else {
                        usrq->list_entry = NULL;
                        sol_ofs_uobj_ref(&usrq->uobj);
                        mutex_exit(&uctxt->lock);
                        rc = uverbs_usrq_free(usrq, uctxt);
                        mutex_enter(&uctxt->lock);
                        if (rc)
                                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                                    "usrq_free(%p) failed", usrq);
                }

                kmem_free((void *)entry, sizeof (genlist_entry_t));
                entry = remove_genlist_head(&uctxt->srq_list);
        }
        (uctxt->srq_list).count = tmp_genlist.count;
        (uctxt->srq_list).head = tmp_genlist.head;
        (uctxt->srq_list).tail = tmp_genlist.tail;

        entry = remove_genlist_head(&uctxt->mr_list);
        while (entry) {
                uverbs_umr_uobj_t *umr = (uverbs_umr_uobj_t *)entry->data;

                rw_enter(&(umr->uobj.uo_lock), RW_WRITER);
                (void) sol_ofs_uobj_remove(&uverbs_umr_uo_tbl, &umr->uobj);
                rw_exit(&(umr->uobj.uo_lock));

                (void) ibt_deregister_mr(uctxt->hca->hdl, umr->mr);
                sol_ofs_uobj_free(&umr->uobj);

                kmem_free((void *)entry, sizeof (genlist_entry_t));
                entry = remove_genlist_head(&uctxt->mr_list);
        }

        entry = remove_genlist_head(&uctxt->pd_list);
        while (entry) {
                uverbs_upd_uobj_t *upd = (uverbs_upd_uobj_t *)entry->data;

                rw_enter(&(upd->uobj.uo_lock), RW_WRITER);
                if (upd->active_qp_cnt) {
                        new_entry = add_genlist(&tmp_genlist, entry->data,
                            entry->data_context);
                        upd->list_entry = new_entry;
                        upd->free_pending = 1;
                        rw_exit(&(upd->uobj.uo_lock));
                } else {
                        upd->list_entry = NULL;
                        sol_ofs_uobj_ref(&upd->uobj);
                        mutex_exit(&uctxt->lock);
                        rc = uverbs_upd_free(upd, uctxt);
                        mutex_enter(&uctxt->lock);
                        if (rc)
                                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                                    "upd_free(%p) failed", upd);
                }

                kmem_free((void *)entry, sizeof (genlist_entry_t));
                entry = remove_genlist_head(&uctxt->pd_list);
        }
        (uctxt->pd_list).count = tmp_genlist.count;
        (uctxt->pd_list).head = tmp_genlist.head;
        (uctxt->pd_list).tail = tmp_genlist.tail;

        mutex_exit(&uctxt->lock);

        /*
         * Release the user file structure to the async file if it
         * has not be released yet. The uctxt for async file will
         * be closed when the async file is closed.
         */
        if (uctxt->async_evfile) {
                uverbs_uctxt_uobj_t     *async_uctxt;

                async_uctxt = uverbs_uobj_get_uctxt_write(
                    uctxt->uctxt_async_id -
                    SOL_UVERBS_DRIVER_MAX_MINOR);
                if (!async_uctxt) {
                        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                            "uverbs_close: Invalid async_id %x",
                            uctxt->uctxt_async_id);
                        sol_ofs_uobj_put(&uctxt->uobj);
                        return (ENXIO);
                }

                async_uctxt->uctxt_verbs_id = 0;
                sol_uverbs_event_file_close(uctxt->async_evfile);
                uctxt->async_evfile = NULL;
                sol_ofs_uobj_put(&async_uctxt->uobj);
        }

        /*
         * Release the write lock and the reference from the get above, and
         * release the reference placed on the user context as process open
         * to release context.
         */
        sol_ofs_uobj_put(&uctxt->uobj);

        /*
         * If some QPs have not been freed, donot free the uctxt.
         * Set uctxt_free_pending flag. This will be freed when
         * the QP will be freed.
         */
        if ((uctxt->qp_list).count) {
                SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
                    "close: uctxt %p, has pending uqp", uctxt);
                uctxt->uctxt_free_pending = 1;
                return (0);
        }

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "close: deallocated user context: %p, ref = %d",
            (void *)uctxt, uctxt->uobj.uo_refcnt);

        sol_ofs_uobj_deref(&uctxt->uobj, sol_ofs_uobj_free);

        return (0);
}

/*
 * Function:
 *      sol_uverbs_read
 * Input:
 *      dev     - Device number.
 *      uiop    - Pointer to the uio structgure where data is to be stored.
 *      credp   - A pointer to the credentials for the I/O transaction.
 * Output:
 *      None
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User process read stub.
 */
static int
sol_uverbs_read(dev_t dev, struct uio *uiop, cred_t *credp)
{
        minor_t                 id = getminor(dev);
        uverbs_uctxt_uobj_t     *uctxt, *verbs_uctxt;
        int                     rc;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "uverbs_read(%x, %p, %p)",
            dev, uiop, credp);

        ASSERT(id >= SOL_UVERBS_DRIVER_MAX_MINOR);
        uctxt = uverbs_uobj_get_uctxt_read(id - SOL_UVERBS_DRIVER_MAX_MINOR);
        if (uctxt == NULL) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_read: Failed get user context");
                return (ENXIO);
        }

        if (uctxt->uctxt_verbs_id < SOL_UVERBS_DRIVER_MAX_MINOR) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_read: Invalid Verbs user context id, %x",
                    uctxt->uctxt_verbs_id);
                sol_ofs_uobj_put(&uctxt->uobj);
                return (ENXIO);
        }
        verbs_uctxt = uverbs_uobj_get_uctxt_read(uctxt->uctxt_verbs_id
            - SOL_UVERBS_DRIVER_MAX_MINOR);
        if (verbs_uctxt == NULL) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_read: Failed get verbs user context");
                sol_ofs_uobj_put(&uctxt->uobj);
                return (ENXIO);
        }
        if (uctxt->uctxt_type == SOL_UVERBS_UCTXT_ASYNC) {
                ASSERT(verbs_uctxt->async_evfile);
                rc = sol_uverbs_event_file_read(verbs_uctxt->async_evfile,
                    uiop, credp);
        } else if (uctxt->uctxt_type == SOL_UVERBS_UCTXT_COMPL) {
                rc = sol_uverbs_event_file_read(uctxt->comp_evfile,
                    uiop, credp);
        } else {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_read: invalid user context type %x",
                    uctxt->uctxt_type);
                rc = ENXIO;
        }

        sol_ofs_uobj_put(&verbs_uctxt->uobj);
        sol_ofs_uobj_put(&uctxt->uobj);
        return (rc);
}

/*
 * Function:
 *      sol_uverbs_mmap
 * Input:
 *      dev             - Device whose memory is to be mapped.
 *      sol_uverbs_mmap - Offset within the device memory at which mapping
 *                        begins.
 *      prot            - Bitmask specifying protection.
 * Output:
 *      None
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User process mmap stub.  Mmap operations are performed directly
 *      by the underlying IB HCA driver, bypassing the user verbs.
 */
/* ARGSUSED */
static int
sol_uverbs_mmap(dev_t dev, off_t mmap_offset, int prot)
{
        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "sol_uverbs_mmap(%d)-  not yet used", mmap_offset);
        return (DDI_SUCCESS);
}

/*
 * Function:
 *      sol_uverbs_get_context
 * Input:
 *      uctxt   - Pointer to the callers user context.
 *      buf     - Pointer to kernel buffer containing command.
 *      in_len  - Length in bytes of input command buffer.
 *      out_len - Length in bytes of output response buffer.
 * Output:
 *      The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verb entry point to return the unique user context to the process
 *      that opened the associated user verb driver instance.  Note that upon
 *      entry a reference will have already been placed on the user
 *      context user space object, so an additional reference is not
 *      required here.
 */
int
sol_uverbs_get_context(uverbs_uctxt_uobj_t *uctxt, char *buf, int in_len,
    int out_len)
{
        struct mthca_alloc_ucontext_resp        uresp;
        struct ib_uverbs_get_context            cmd;
        struct ib_uverbs_get_context_resp       resp;
        struct ib_udata                         udata;
        int                                     rc;
        minor_t                                 async_id;
        uverbs_uctxt_uobj_t                     *async_uctxt;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "uverbs_get_context() - buf %p, sizeof (cmd) %d",
            buf, sizeof (cmd));

        ASSERT(uctxt->hca);

        (void) memcpy(&cmd, buf, sizeof (cmd));

        udata.inbuf  = (void *)(buf + sizeof (cmd));
#ifdef  _LP64
        udata.outbuf = (void *)(cmd.response.r_laddr + sizeof (resp));
#else
        udata.outbuf = (void *)(cmd.response.r_addr + sizeof (resp));
#endif
        udata.inlen  = in_len - sizeof (cmd);
        udata.outlen = out_len - sizeof (resp);

        /*
         * libibverbs will have passed minor of the async file in
         * resp.fd. Use this to determine the uctxt created for
         * asyncs.
         */
#ifdef  _LP64
        rc = copyin((void*)cmd.response.r_laddr, (void*)&resp, sizeof (resp));
#else
        rc = copyin((void*)cmd.response.r_addr, (void*)&resp, sizeof (resp));
#endif
        if (rc != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "get_context: copyin (rc=%d)", rc);
                rc = EFAULT;
                goto out;
        }
        async_id = resp.async_fd;
        if (async_id < SOL_UVERBS_DRIVER_MAX_MINOR) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "get_context: Invalid async user context "
                    "id %x", async_id);
                return (ENXIO);
        }

        async_uctxt = uverbs_uobj_get_uctxt_read(async_id -
            SOL_UVERBS_DRIVER_MAX_MINOR);
        if (async_uctxt == NULL) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "get_context: Failed get async user context");
                return (ENXIO);
        }
        if (async_uctxt->uctxt_type != SOL_UVERBS_UCTXT_EVENT ||
            async_uctxt->uctxt_verbs_id != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "get_context: Invalid user context - "
                    "possibly reused");
                return (ENXIO);
        }
        async_uctxt->uctxt_type = SOL_UVERBS_UCTXT_ASYNC;
        async_uctxt->uctxt_verbs_id = uctxt->uobj.uo_id +
            SOL_UVERBS_DRIVER_MAX_MINOR;
        uctxt->uctxt_async_id = async_id;
        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "get_context: uctxt %p, async_uctxt %p, async_id %x",
            uctxt, async_uctxt, async_id);
        sol_ofs_uobj_put(&async_uctxt->uobj);

        uctxt->async_evfile = uverbs_alloc_event_file(uctxt, 1);
        if (!uctxt->async_evfile) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "get_context: async event file allocation failed");
                goto out;
        }

        (void) memset(&resp, 0, sizeof (resp));
        resp.num_comp_vectors   = 1;

#ifdef  _LP64
        rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
#else
        rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
#endif
        if (rc != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "get_context: copyout (rc=%d)", rc);
                rc = EFAULT;
                goto out;
        }

        /*
         * This unfortunately is Mellanox specific, we need to consider moving
         * this directly into the command response as opaque data, instead of
         * using this method.
         */
        (void) memset(&uresp, 0, sizeof (uresp));
        uresp.uarc_size   = 0;
        uresp.qp_tab_size = uctxt->hca->attr.hca_max_chans;

        rc = copyout((void*)&uresp, (void*)udata.outbuf, sizeof (uresp));
        if (rc != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "get_context: copyout outbuf (rc=%d)", rc);
                rc = EFAULT;
                goto out;
        }
        rc = DDI_SUCCESS;

out:
        return (rc);
}

/*
 * Function:
 *      sol_uverbs_alloc_pd
 * Input:
 *      uctxt   - Pointer to the callers user context.
 *      buf     - Pointer to kernel buffer containing a alloc PD command.
 *      in_len  - Length in bytes of input command buffer.
 *      out_len - Length in bytes of output response buffer.
 * Output:
 *      The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verb entry point to allocate a device protection domain.
 */
/* ARGSUSED */
int
sol_uverbs_alloc_pd(uverbs_uctxt_uobj_t *uctxt, char *buf, int in_len,
    int out_len)
{
        struct ib_uverbs_alloc_pd       cmd;
        struct ib_uverbs_alloc_pd_resp  resp;
        uverbs_upd_uobj_t               *upd;
        int                             rc;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "alloc_pd()");

        (void) memcpy(&cmd, buf, sizeof (cmd));
        (void) memset(&resp, 0, sizeof (resp));

        upd = kmem_zalloc(sizeof (*upd), KM_NOSLEEP);
        if (upd == NULL) {
                rc = ENOMEM;
                goto out;
        }
        sol_ofs_uobj_init(&upd->uobj, 0, SOL_UVERBS_UPD_UOBJ_TYPE);
        rw_enter(&upd->uobj.uo_lock, RW_WRITER);

        rc = ibt_alloc_pd(uctxt->hca->hdl, IBT_PD_NO_FLAGS, &upd->pd);
        if (rc != IBT_SUCCESS) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "alloc_pd: ibt_alloc_pd() (rc=%d)", rc);
                rc = sol_uverbs_ibt_to_kernel_status(rc);
                upd->uobj.uo_uobj_sz = sizeof (uverbs_upd_uobj_t);
                goto alloc_err;
        }

        if (sol_ofs_uobj_add(&uverbs_upd_uo_tbl, &upd->uobj) != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "alloc_pd: User object add failed");
                rc = ENOMEM;
                goto err_add_uobj;
        }
        resp.pd_handle = upd->uobj.uo_id;

        /*
         * Query underlying hardware driver for data that may be required
         * when using the PD in an OS Bypass creation of UD address vectors.
         */
        rc = ibt_ci_data_out(uctxt->hca->hdl, IBT_CI_NO_FLAGS, IBT_HDL_PD,
            (void *)upd->pd, &resp.drv_out, sizeof (resp.drv_out));
        if (rc != IBT_SUCCESS) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "alloc_pd: ibt_ci_data_out() (rc=%d)", rc);
                rc = EFAULT;
                goto err_response;
        }

#ifdef  _LP64
        rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
#else
        rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
#endif
        if (rc != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "alloc_pd: copyout fail (rc=%d)", rc);
                rc = EFAULT;
                goto err_response;
        }

        mutex_enter(&uctxt->lock);
        upd->list_entry = add_genlist(&uctxt->pd_list, (uintptr_t)upd, uctxt);
        mutex_exit(&uctxt->lock);

        if (!upd->list_entry) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "alloc_pd: Error adding upd to pd_list\n");
                rc = ENOMEM;
                goto err_response;
        }

        upd->uobj.uo_live = 1;
        rw_exit(&upd->uobj.uo_lock);
        return (DDI_SUCCESS);

err_response:
        /*
         * Need to set uo_live, so sol_ofs_uobj_remove() will
         * remove the object from the object table.
         */
        upd->uobj.uo_live = 1;
        (void) sol_ofs_uobj_remove(&uverbs_upd_uo_tbl, &upd->uobj);

err_add_uobj:
        (void) ibt_free_pd(uctxt->hca->hdl, upd->pd);

alloc_err:
        rw_exit(&upd->uobj.uo_lock);
        sol_ofs_uobj_deref(&upd->uobj, sol_ofs_uobj_free);
out:
        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "alloc_pd:error (rc=%d)", rc);
        return (rc);
}

int
uverbs_upd_free(uverbs_upd_uobj_t *upd, uverbs_uctxt_uobj_t *uctxt)
{
        int     rc;

        rc = ibt_free_pd(uctxt->hca->hdl, upd->pd);
        if (rc != IBT_SUCCESS) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_upd_free: ibt_free_pd() failed %d", rc);
                rc = sol_uverbs_ibt_to_kernel_status(rc);
                sol_ofs_uobj_put(&upd->uobj);
                return (rc);
        }

        /*
         * Remove from the list of this contexts PD resources, then remove from
         * the resource managment table and the reference placed on the user
         * object at PD allocation.
         */
        upd->pd = NULL;
        if (upd->list_entry) {
                mutex_enter(&uctxt->lock);
                delete_genlist(&uctxt->pd_list, upd->list_entry);
                mutex_exit(&uctxt->lock);
        }

        /*
         * list_entry is NULL when called from sol_uverbs_close. Remove
         * from upd_uo_tbl and free upd, when called from close also.
         */
        sol_ofs_uobj_put(&upd->uobj);
        (void) sol_ofs_uobj_remove(&uverbs_upd_uo_tbl, &upd->uobj);
        sol_ofs_uobj_deref(&upd->uobj, sol_ofs_uobj_free);
        return (0);
}

/*
 * Function:
 *      sol_uverbs_dealloc_pd
 * Input:
 *      uctxt   - Pointer to the callers user context.
 *      buf     - Pointer to kernel buffer containing dealloc PD command.
 *      in_len  - Length in bytes of input command buffer.
 *      out_len - Length in bytes of output response buffer.
 * Output:
 *      The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verb entry point to de-allocate a device protection domain.
 */
/* ARGSUSED */
int
sol_uverbs_dealloc_pd(uverbs_uctxt_uobj_t *uctxt, char *buf, int in_len,
    int out_len)
{
        struct ib_uverbs_dealloc_pd     cmd;
        uverbs_upd_uobj_t               *upd;
        int                             rc = 0;

        (void) memcpy(&cmd, buf, sizeof (cmd));

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "dealloc_pd(%d)", cmd.pd_handle);

        upd = uverbs_uobj_get_upd_write(cmd.pd_handle);
        if (upd == NULL) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "dealloc_pd(%d) : invalid hdl", cmd.pd_handle);
                rc = EINVAL;
                goto err_out1;
        }

        if (upd->active_qp_cnt) {
                sol_ofs_uobj_put(&upd->uobj);
                rc = EBUSY;
        } else {
                rc = uverbs_upd_free(upd, uctxt);
        }
        cmd.pd_handle = 0;
        return (rc);

err_out1:
        return (rc);
}

/*
 * Function:
 *      sol_uverbs_query_device
 * Input:
 *      uctxt   - Pointer to the callers user context.
 *      buf     - Pointer to kernel buffer containing query device command.
 *      in_len  - Length in bytes of input command buffer.
 *      out_len - Length in bytes of output response buffer.
 * Output:
 *      The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verb entry point to query device attributes.
 */
/* ARGSUSED */
int
sol_uverbs_query_device(uverbs_uctxt_uobj_t *uctxt, char *buf, int in_len,
        int out_len)
{
        struct ib_uverbs_query_device           cmd;
        struct ib_uverbs_query_device_resp      resp;
        ibt_hca_attr_t                          hca_attr;
        int                                     rc;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "query_device()");

        (void) memcpy(&cmd, buf, sizeof (cmd));
        rc = ibt_query_hca(uctxt->hca->hdl, &hca_attr);
        if (rc != IBT_SUCCESS) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "query_device: ibt_query_hca() (rc=%d)", rc);
                rc = sol_uverbs_ibt_to_kernel_status(rc);
                goto out;
        }

        (void) memset(&resp, 0, sizeof (resp));

        resp.fw_ver = ((uint64_t)hca_attr.hca_fw_major_version << 32) |
            ((uint64_t)hca_attr.hca_fw_minor_version << 16) |
            ((uint64_t)hca_attr.hca_fw_micro_version);

        /*
         * NOTE: node guid and system image guid must be returned in big
         * endian (network order).  On solaris these are in host
         * order, so we swap it back here.
         */
        resp.node_guid                  = htonll(hca_attr.hca_node_guid);
        resp.sys_image_guid             = htonll(hca_attr.hca_si_guid);

        resp.max_mr_size                = hca_attr.hca_max_memr_len;

        resp.page_size_cap =
            sol_uverbs_ibt_to_of_page_sz(hca_attr.hca_page_sz);

        resp.vendor_id                  = hca_attr.hca_vendor_id;
        resp.vendor_part_id             = hca_attr.hca_device_id;
        resp.hw_ver                     = hca_attr.hca_version_id;
        resp.max_qp                     = hca_attr.hca_max_chans;
        resp.max_qp_wr                  = hca_attr.hca_max_chan_sz;

        resp.device_cap_flags           =
            sol_uverbs_ibt_to_of_device_cap_flags(hca_attr.hca_flags,
            hca_attr.hca_flags2);

        resp.max_sge                    = hca_attr.hca_max_sgl;
        resp.max_sge_rd                 = hca_attr.hca_max_sgl;
        resp.max_cq                     = hca_attr.hca_max_cq;
        resp.max_cqe                    = hca_attr.hca_max_cq_sz;
        resp.max_mr                     = hca_attr.hca_max_memr;
        resp.max_pd                     = hca_attr.hca_max_pd;
        resp.max_qp_rd_atom             = hca_attr.hca_max_rdma_in_chan;
        resp.max_ee_rd_atom             = 0;
        resp.max_res_rd_atom            = hca_attr.hca_max_rsc;
        resp.max_qp_init_rd_atom        = hca_attr.hca_max_rdma_out_chan;
        resp.max_ee_init_rd_atom        = 0;
        if (hca_attr.hca_flags & IBT_HCA_ATOMICS_GLOBAL) {
                resp.atomic_cap = IB_ATOMIC_GLOB;
        } else if (hca_attr.hca_flags & IBT_HCA_ATOMICS_HCA) {
                resp.atomic_cap = IB_ATOMIC_HCA;
        } else {
                resp.atomic_cap = IB_ATOMIC_NONE;
        }
        resp.max_ee                     = 0;
        resp.max_rdd                    = 0;
        resp.max_mw                     = hca_attr.hca_max_mem_win;
        resp.max_raw_ipv6_qp            = hca_attr.hca_max_ipv6_chan;
        resp.max_raw_ethy_qp            = hca_attr.hca_max_ether_chan;
        resp.max_mcast_grp              = hca_attr.hca_max_mcg;
        resp.max_mcast_qp_attach        = hca_attr.hca_max_chan_per_mcg;
        resp.max_total_mcast_qp_attach  = hca_attr.hca_max_mcg_chans;
        resp.max_ah                     = hca_attr.hca_max_ud_dest;
        resp.max_fmr                    = hca_attr.hca_max_fmrs;
        resp.max_map_per_fmr            = 0;
        resp.max_srq                    = hca_attr.hca_max_srqs;
        resp.max_srq_wr                 = hca_attr.hca_max_srqs_sz;
        resp.max_srq_sge                = hca_attr.hca_max_srq_sgl;
        resp.max_pkeys                  = hca_attr.hca_max_port_pkey_tbl_sz;
        resp.local_ca_ack_delay         = hca_attr.hca_local_ack_delay;
        resp.phys_port_cnt              = hca_attr.hca_nports;

#ifdef  _LP64
        rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
#else
        rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
#endif
        if (rc != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "query_device: Error writing resp data (rc=%d)", rc);
                rc = EFAULT;
                goto out;
        }

        rc = DDI_SUCCESS;

out:
        return (rc);
}

/*
 * Function:
 *      sol_uverbs_query_port
 * Input:
 *      uctxt   - Pointer to the callers user context.
 *      buf     - Pointer to kernel buffer containing query port command.
 *      in_len  - Length in bytes of input command buffer.
 *      out_len - Length in bytes of output response buffer.
 * Output:
 *      The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verb entry point to query a device port attributes.
 */
/* ARGSUSED */
int
sol_uverbs_query_port(uverbs_uctxt_uobj_t *uctxt, char *buf, int in_len,
    int out_len)
{
        struct ib_uverbs_query_port             cmd;
        struct ib_uverbs_query_port_resp        resp;
        ibt_hca_portinfo_t                      *port_info;
        uint_t                                  port_info_n;
        uint_t                                  port_info_size;
        int                                     rc;

        (void) memcpy(&cmd, buf, sizeof (cmd));

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "sol_uverbs_query_port: %d",
            cmd.port_num);

        if (!cmd.port_num || cmd.port_num > uctxt->hca->attr.hca_nports) {
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "query_port: Invalid port specified");

                rc = EINVAL;
                goto out;
        }

        rc = ibt_query_hca_ports(uctxt->hca->hdl, cmd.port_num, &port_info,
            &port_info_n, &port_info_size);

        if (rc != IBT_SUCCESS) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "query_port: ibt_query_hca_ports() (rc=%d)", rc);
                rc = sol_uverbs_ibt_to_kernel_status(rc);
                goto out;
        }
        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "query_port: "
            "port_num %d, port_info %x, lid %x, sm_lid %x",
            cmd.port_num, port_info, port_info->p_opaque1,
            port_info->p_sm_lid);

        (void) memset(&resp, 0, sizeof (resp));

        resp.state                      = port_info->p_linkstate;
        resp.max_mtu                    = port_info->p_mtu;
        resp.active_mtu                 = port_info->p_mtu;
        resp.gid_tbl_len                = port_info->p_sgid_tbl_sz;
        resp.port_cap_flags             = port_info->p_capabilities;
        resp.max_msg_sz                 = port_info->p_msg_sz;
        resp.bad_pkey_cntr              = port_info->p_pkey_violations;
        resp.qkey_viol_cntr             = port_info->p_qkey_violations;
        resp.pkey_tbl_len               = port_info->p_pkey_tbl_sz;
        resp.lid                        = port_info->p_opaque1;
        resp.sm_lid                     = port_info->p_sm_lid;
        resp.lmc                        = port_info->p_lmc;
        resp.max_vl_num                 = port_info->p_max_vl;
        resp.sm_sl                      = port_info->p_sm_sl;
        resp.subnet_timeout             = port_info->p_subnet_timeout;
        resp.init_type_reply            = port_info->p_init_type_reply;
        resp.active_width               = port_info->p_width_active;
        resp.active_speed               = port_info->p_speed_active;
        resp.phys_state                 = port_info->p_phys_state;

        ibt_free_portinfo(port_info, port_info_size);

#ifdef  _LP64
        rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
#else
        rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
#endif
        if (rc != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "query_port : copyout fail %x", rc);
                rc = EFAULT;
                goto out;
        }

        rc = DDI_SUCCESS;

out:
        return (rc);
}

/*
 * Function:
 *      sol_uverbs_query_gid
 * Input:
 *      uctxt   - Pointer to the callers user context.
 *      buf     - Pointer to kernel buffer containing query gid command.
 *      in_len  - Length in bytes of input command buffer.
 *      out_len - Length in bytes of output response buffer.
 * Output:
 *      The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verb entry point to query the device gid for the specified
 *      port and gid index.
 */
/* ARGSUSED */
int
sol_uverbs_query_gid(uverbs_uctxt_uobj_t *uctxt, char *buf, int in_len,
    int out_len)
{
        struct ib_uverbs_query_gid      cmd;
        struct ib_uverbs_query_gid_resp resp;
        ibt_hca_portinfo_t              *port_info;
        uint_t                          port_info_n;
        uint_t                          port_info_size;
        int                             rc;
        uint64_t                        temp;

        (void) memcpy(&cmd, buf, sizeof (cmd));

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "query_gid() : port_num %x, gid_index %x",
            cmd.port_num, cmd.gid_index);

        if (!cmd.port_num || cmd.port_num > uctxt->hca->attr.hca_nports) {
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "query_gid: Invalid port specified");

                rc = EINVAL;
                goto out;
        }

        rc = ibt_query_hca_ports(uctxt->hca->hdl, cmd.port_num, &port_info,
            &port_info_n, &port_info_size);
        if (rc != IBT_SUCCESS) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "query_gid: ibt_query_hca_ports() (rc=%d)", rc);
                rc = sol_uverbs_ibt_to_kernel_status(rc);
                goto out;
        }

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "number of "
            "gid entries %d", cmd.port_num, cmd.gid_index,
            port_info->p_sgid_tbl_sz);


        if (cmd.gid_index >= port_info->p_sgid_tbl_sz) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "query_gid: cmd gid_index %x > port_info sz %x",
                    cmd.gid_index, port_info->p_sgid_tbl_sz);
                rc = EINVAL;
                ibt_free_portinfo(port_info, port_info_size);
                goto out;
        }

        /*
         * The gid must be returned as a network ordered byte array, on solaris
         * it is a structure in host order so we swap the components as needed.
         */
        temp = htonll(port_info->p_sgid_tbl[cmd.gid_index].gid.ucast_gid.
            ugid_prefix);
        (void) memcpy(&resp.gid[0], &temp, sizeof (temp));
        temp = htonll(port_info->p_sgid_tbl[cmd.gid_index].gid.ucast_gid.
            ugid_guid);
        (void) memcpy(&resp.gid[8], &temp, sizeof (temp));

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "query_gid: gid = "
            "0x%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:"
            "%02x:%02x:%02x:%02x:%02x:%02x:%02x:%02x:",
            resp.gid[0], resp.gid[1], resp.gid[2], resp.gid[3],
            resp.gid[4], resp.gid[5], resp.gid[6], resp.gid[7],
            resp.gid[8], resp.gid[9], resp.gid[10], resp.gid[11],
            resp.gid[12], resp.gid[13], resp.gid[14], resp.gid[15]);

        ibt_free_portinfo(port_info, port_info_size);

#ifdef  _LP64
        rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
#else
        rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
#endif
        if (rc != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "query_gid: copyout %d", rc);
                rc = EFAULT;
                goto out;
        }

        rc = DDI_SUCCESS;
out:
        return (rc);
}

/*
 * Function:
 *      sol_uverbs_query_pkey
 * Input:
 *      uctxt   - Pointer to the callers user context.
 *      buf     - Pointer to kernel buffer containing a query pkey command.
 *      in_len  - Length in bytes of input command buffer.
 *      out_len - Length in bytes of output response buffer.
 * Output:
 *      The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verb entry point to query a device for the pkey at the specified
 *      port and pkey index.
 */
/* ARGSUSED */
int
sol_uverbs_query_pkey(uverbs_uctxt_uobj_t *uctxt, char *buf, int in_len,
    int out_len)
{
        struct ib_uverbs_query_pkey             cmd;
        struct ib_uverbs_query_pkey_resp        resp;
        ibt_hca_portinfo_t                      *port_info;
        uint_t                                  port_info_n;
        uint_t                                  port_info_size;
        int                                     rc;

        (void) memcpy(&cmd, buf, sizeof (cmd));

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "query_pkey: entry, port = %d, pkey index = %d",
            cmd.port_num, cmd.pkey_index);

        if (!cmd.port_num || cmd.port_num > uctxt->hca->attr.hca_nports) {
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "query_pkey: Invalid port specified");

                rc = EINVAL;
                goto out;
        }

        rc = ibt_query_hca_ports(uctxt->hca->hdl, cmd.port_num, &port_info,
            &port_info_n, &port_info_size);
        if (rc != IBT_SUCCESS) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "query_pkey: ibt_query_hca_ports() %d", rc);
                rc = sol_uverbs_ibt_to_kernel_status(rc);
                goto out;
        }

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "query_pkey: port %d, requested index %d, number of pkey entries "
            "%d", cmd.port_num, cmd.pkey_index, port_info->p_pkey_tbl_sz);


        if (cmd.pkey_index >= port_info->p_pkey_tbl_sz) {
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "query_pkey: Invalid index %d, table size = %d",
                    cmd.pkey_index, port_info->p_pkey_tbl_sz);

                ibt_free_portinfo(port_info, port_info_size);
                rc = EINVAL;
                goto out;
        }

        (void) memset(&resp, 0, sizeof (resp));
        resp.pkey = port_info->p_pkey_tbl[cmd.pkey_index];

        ibt_free_portinfo(port_info, port_info_size);

#ifdef  _LP64
        rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
#else
        rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
#endif
        if (rc != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "query_pkey: copyout %d", rc);
                rc = EFAULT;
                goto out;
        }

        rc = DDI_SUCCESS;
out:
        return (rc);
}

/*
 * Function:
 *      sol_uverbs_reg_mr
 * Input:
 *      uctxt   - Pointer to the callers user context.
 *      buf     - Pointer to kernel buffer containing command.
 *      in_len  - Length in bytes of input command buffer.
 *      out_len - Length in bytes of output response buffer.
 * Output:
 *      The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verb entry point to register a memory region.
 */
/* ARGSUSED */
int
sol_uverbs_reg_mr(uverbs_uctxt_uobj_t *uctxt, char *buf, int in_len,
    int out_len)
{
        struct ib_uverbs_reg_mr         cmd;
        struct ib_uverbs_reg_mr_resp    resp;
        uverbs_upd_uobj_t               *upd;
        uverbs_umr_uobj_t               *umr;
        ibt_mr_attr_t                   new_mem_attr;
        ibt_mr_desc_t                   new_mr_desc;
        int                             rc;

        (void) memcpy(&cmd, buf, sizeof (cmd));
        (void) memset(&resp, 0, sizeof (resp));
        (void) memset(&new_mem_attr, 0, sizeof (new_mem_attr));
        (void) memset(&new_mr_desc, 0, sizeof (new_mr_desc));

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "reg_mr()");

        new_mem_attr.mr_vaddr   = cmd.start;
        new_mem_attr.mr_len     = cmd.length;
        new_mem_attr.mr_as      = curproc->p_as;
        new_mem_attr.mr_flags   = IBT_MR_NOSLEEP;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "reg_mr : "
            "mr_vaddr 0x%0lX, mr_len %d, mr_as %d, mr_flags %d",
            new_mem_attr.mr_vaddr, new_mem_attr.mr_len,
            new_mem_attr.mr_as, new_mem_attr.mr_flags);

        if ((cmd.access_flags & IB_ACCESS_LOCAL_WRITE) ==
            IB_ACCESS_LOCAL_WRITE) {
                new_mem_attr.mr_flags |= IBT_MR_ENABLE_LOCAL_WRITE;
        }
        if ((cmd.access_flags & IB_ACCESS_REMOTE_WRITE) ==
            IB_ACCESS_REMOTE_WRITE) {
                new_mem_attr.mr_flags |= IBT_MR_ENABLE_REMOTE_WRITE;
        }
        if ((cmd.access_flags & IB_ACCESS_REMOTE_READ) ==
            IB_ACCESS_REMOTE_READ) {
                new_mem_attr.mr_flags |= IBT_MR_ENABLE_REMOTE_READ;
        }
        if ((cmd.access_flags & IB_ACCESS_REMOTE_ATOMIC) ==
            IB_ACCESS_REMOTE_ATOMIC) {
                new_mem_attr.mr_flags |= IBT_MR_ENABLE_REMOTE_ATOMIC;
        }
        if ((cmd.access_flags & IB_ACCESS_MW_BIND) == IB_ACCESS_MW_BIND) {
                new_mem_attr.mr_flags |= IBT_MR_ENABLE_WINDOW_BIND;
        }
        if ((cmd.access_flags & IB_ACCESS_SO) == IB_ACCESS_SO) {
                new_mem_attr.mr_flags |= IBT_MR_DISABLE_RO;
        }

        umr = kmem_zalloc(sizeof (*umr), KM_NOSLEEP);
        if (umr == NULL) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "reg_mr: User object mem allocation error");
                rc = ENOMEM;
                goto out;
        }
        sol_ofs_uobj_init(&umr->uobj, 0, SOL_UVERBS_UMR_UOBJ_TYPE);
        rw_enter(&umr->uobj.uo_lock, RW_WRITER);

        upd = uverbs_uobj_get_upd_read(cmd.pd_handle);
        if (upd == NULL) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "reg_mr: PD invalid");
                rc = EINVAL;
                goto bad_pd;
        }

        rc = ibt_register_mr(uctxt->hca->hdl, upd->pd, &new_mem_attr, &umr->mr,
            &new_mr_desc);

        if (rc != IBT_SUCCESS) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "reg_mr: ibt_register_mr() (rc=%d)", rc);
                rc = sol_uverbs_ibt_to_kernel_status(rc);
                umr->uobj.uo_uobj_sz = sizeof (uverbs_umr_uobj_t);
                goto err_register;
        }

        if (sol_ofs_uobj_add(&uverbs_umr_uo_tbl, &umr->uobj) != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "reg_mr: User object add failed");
                rc = ENOMEM;
                goto err_add_uobj;
        }

        resp.mr_handle  = umr->uobj.uo_id;
        resp.lkey       = new_mr_desc.md_lkey;
        resp.rkey       = new_mr_desc.md_rkey;

#ifdef  _LP64
        rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
#else
        rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
#endif
        if (rc != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "reg_mr: Error writing resp data (rc=%d)", rc);
                rc = EFAULT;
                goto err_response;
        }

        mutex_enter(&uctxt->lock);
        umr->list_entry  = add_genlist(&uctxt->mr_list, (uintptr_t)umr, uctxt);
        mutex_exit(&uctxt->lock);

        if (!umr->list_entry) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "reg_mr: Error adding umr to mr_list\n");
                rc = ENOMEM;
                goto err_response;
        }

        umr->uobj.uo_live = 1;
        rw_exit(&umr->uobj.uo_lock);

        sol_ofs_uobj_put(&upd->uobj);

        return (DDI_SUCCESS);

err_response:
        /*
         * Need to set uo_live, so sol_ofs_uobj_remove() will
         * remove the object from the object table.
         */
        umr->uobj.uo_live = 1;
        (void) sol_ofs_uobj_remove(&uverbs_umr_uo_tbl, &umr->uobj);

err_add_uobj:
        (void) ibt_deregister_mr(uctxt->hca->hdl, umr->mr);

err_register:
        sol_ofs_uobj_put(&upd->uobj);

bad_pd:
        rw_exit(&umr->uobj.uo_lock);
        sol_ofs_uobj_deref(&umr->uobj, sol_ofs_uobj_free);

out:
        return (rc);
}

/*
 * Function:
 *      sol_uverbs_dereg_mr
 * Input:
 *      uctxt   - Pointer to the callers user context.
 *      buf     - Pointer to kernel buffer containing command.
 *      in_len  - Length in bytes of input command buffer.
 *      out_len - Length in bytes of output response buffer.
 * Output:
 *      The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verb entry point to de-register a memory region.
 */
/* ARGSUSED */
int
sol_uverbs_dereg_mr(uverbs_uctxt_uobj_t *uctxt, char *buf, int in_len,
    int out_len)
{
        struct ib_uverbs_dereg_mr       cmd;
        uverbs_umr_uobj_t               *umr;
        int                             rc;

        (void) memcpy(&cmd, buf, sizeof (cmd));

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "dereg_mr(mr_handle=%d)", cmd.mr_handle);

        umr = uverbs_uobj_get_umr_write(cmd.mr_handle);
        if (umr == NULL) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "dereg_mr: Invalid handle");
                rc = EINVAL;
                goto err_out;
        }

        rc = ibt_deregister_mr(uctxt->hca->hdl, umr->mr);

        if (rc != IBT_SUCCESS) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "dereg_mr: ibt_deregister_mr() (rc=%d)", rc);
                rc = sol_uverbs_ibt_to_kernel_status(rc);
                goto err_deregister;
        }

        /*
         * Remove from the list of this contexts MR resources, then remove from
         * the resource management table and the reference placed on the user
         * object at MR creation.
         */
        mutex_enter(&uctxt->lock);
        delete_genlist(&uctxt->mr_list, umr->list_entry);
        mutex_exit(&uctxt->lock);

        (void) sol_ofs_uobj_remove(&uverbs_umr_uo_tbl, &umr->uobj);

        /*
         * Drop the lock and ref held by get_umr_write.
         */
        sol_ofs_uobj_put(&umr->uobj);

        sol_ofs_uobj_deref(&umr->uobj, sol_ofs_uobj_free);

        cmd.mr_handle = 0;
        return (DDI_SUCCESS);

err_deregister:
        /*
         * Drop the lock and ref held by get_umr_write.
         */
        sol_ofs_uobj_put(&umr->uobj);

err_out:
        return (rc);
}

/*
 * Function:
 *      sol_uverbs_create_ah
 * Input:
 *      uctxt   - Pointer to the callers user context.
 *      buf     - Pointer to kernel buffer containing command.
 *      in_len  - Length in bytes of input command buffer.
 *      out_len - Length in bytes of output response buffer.
 * Output:
 *      The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verb entry point to for devices that require kernel AH creation.
 */
/* ARGSUSED */
int
sol_uverbs_create_ah(uverbs_uctxt_uobj_t *uctxt, char *buf, int in_len,
    int out_len)
{
        SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
            "create_ah: kernel user verb not implemented");
        return (ENOTSUP);
}

/*
 * Function:
 *      sol_uverbs_destroy_ah
 * Input:
 *      uctxt   - Pointer to the callers user context.
 *      buf     - Pointer to kernel buffer containing command.
 *      in_len  - Length in bytes of input command buffer.
 *      out_len - Length in bytes of output response buffer.
 * Output:
 *      The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verb entry point to for devices that require kernel AH deletion.
 */
/* ARGSUSED */
int
sol_uverbs_destroy_ah(uverbs_uctxt_uobj_t *uctxt, char *buf, int in_len,
    int out_len)
{
        SOL_OFS_DPRINTF_L3(sol_uverbs_dbg_str,
            "destroy_ah: kernel user verb not implemented");
        return (ENOTSUP);
}

/*
 * Function:
 *      sol_uverbs_create_comp_chan
 * Input:
 *      uctxt   - Pointer to the callers user context.
 *      buf     - Pointer to kernel buffer containing command.
 *      in_len  - Length in bytes of input command buffer.
 *      out_len - Length in bytes of output response buffer.
 * Output:
 *      The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verb entry point to create a completion event channel.
 */
int
sol_uverbs_create_comp_channel(uverbs_uctxt_uobj_t *uctxt, char *buf,
    int in_len, int out_len)
{
        struct ib_uverbs_create_comp_channel            cmd;
        struct ib_uverbs_create_comp_channel_resp       resp;
        int                                             rc;
        minor_t                                         compl_id;
        uverbs_uctxt_uobj_t                             *compl_uctxt;

        (void) memcpy(&cmd, buf, sizeof (cmd));
        (void) memset(&resp, 0, sizeof (resp));

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "create_comp_chan: entry, in_len=%d, out_len=%d",
            in_len, out_len);

        /*
         * libibverbs will have passed minor of the compl file in
         * resp.fd. Use this to determine the uctxt created for
         * completions.
         */
#ifdef  _LP64
        rc = copyin((void*)cmd.response.r_laddr, (void*)&resp, sizeof (resp));
#else
        rc = copyin((void*)cmd.response.r_addr, (void*)&resp, sizeof (resp));
#endif
        if (rc != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "create_comp: copyin (rc=%d)", rc);
                rc = EFAULT;
                return (rc);
        }
        compl_id = resp.fd;
        if (compl_id < SOL_UVERBS_DRIVER_MAX_MINOR) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "create_comp: Invalid compl user context id %x",
                    compl_id);
                return (ENXIO);
        }

        compl_uctxt = uverbs_uobj_get_uctxt_read(compl_id -
            SOL_UVERBS_DRIVER_MAX_MINOR);
        if (compl_uctxt == NULL) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "create_comp: Failed get compl user context");
                return (ENXIO);
        }
        if (compl_uctxt->uctxt_type != SOL_UVERBS_UCTXT_EVENT ||
            compl_uctxt->uctxt_verbs_id != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "create_comp_chan: Invalid user context - "
                    "possibly reused");
                return (ENXIO);
        }
        compl_uctxt->uctxt_type = SOL_UVERBS_UCTXT_COMPL;
        compl_uctxt->uctxt_verbs_id = uctxt->uobj.uo_id +
            SOL_UVERBS_DRIVER_MAX_MINOR;
        uctxt->uctxt_comp_id = compl_id;
        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "create_comp_chan: "
            "uctxt %p, compl_uctxt %p, compl_id %x", uctxt,
            compl_uctxt, compl_id);

        /*
         * Allocate an event file to be used for completion
         * event notification.
         */
        compl_uctxt->comp_evfile = uverbs_alloc_event_file(uctxt, 0);
        if (compl_uctxt->comp_evfile == NULL) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "create_comp_chan: Event file alloc error");
                rc = EINVAL;
                sol_ofs_uobj_put(&compl_uctxt->uobj);
                return (rc);
        }

        /*
         * Place an extra reference on the compl event file.  These will
         * be used to handle the natural race of between the closing of
         * the compl event file and uverbs device file that can occur.
         */
        sol_ofs_uobj_ref(&compl_uctxt->comp_evfile->uobj);

        sol_ofs_uobj_put(&compl_uctxt->uobj);

#ifdef  _LP64
        rc = copyout((void*)&resp, (void*)cmd.response.r_laddr, sizeof (resp));
#else
        rc = copyout((void*)&resp, (void*)cmd.response.r_addr, sizeof (resp));
#endif
        if (rc != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "create_comp_chan: copyout %d", rc);
                rc = EFAULT;
                return (rc);
        }

        return (0);
}

/*
 * Function:
 *      sol_uverbs_dummy_command
 * Input:
 *      uctxt   - Pointer to the callers user context.
 *      buf     - Pointer to kernel buffer containing command.
 *      in_len  - Length in bytes of input command buffer.
 *      out_len - Length in bytes of output response buffer.
 * Output:
 *      The command output buffer is updated with command results.
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verb generic place holder stub.
 */
/* ARGSUSED */
int
sol_uverbs_dummy_command(uverbs_uctxt_uobj_t *uctxt, char *buf, int in_len,
    int out_len)
{
        SOL_OFS_DPRINTF_L4(sol_uverbs_dbg_str,
            "sol_uverbs_dummy_command invoked");

        return (0);
}

/*
 * Function:
 *      sol_uverbs_write
 * Input:
 *      dev     - Device number.
 *      uiop    - Pointer to the uio structure that describes the data (i.e.
 *                Solaris User Verbs command).
 *      credp   - A pointer to the user credentials for the I/O transaction.
 * Output:
 *      uiop    -
 * Returns:
 *      DDI_SUCCESS on success, else error code.
 * Description:
 *      User verb write entry point.  A user deivce libraries use this
 *      entry point to execute a kernel agent user verbs call.  During
 *      the course of the call the user process will hold a read reference
 *      to the associated user context.
 */
#define SOL_UVERBS_MAX_CMD_PAYLOAD    512
/* ARGSUSED */
static int
sol_uverbs_write(dev_t dev, struct uio *uiop, cred_t *credp)
{
        uverbs_uctxt_uobj_t             *uctxt;
        size_t                          len = uiop->uio_resid;
        int                             rc;
        struct ib_uverbs_cmd_hdr        hdr;
        char                            payload[SOL_UVERBS_MAX_CMD_PAYLOAD];
        minor_t                         id = getminor(dev);

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "uverbs_write: entry (len=%d)", len);

        ASSERT(id >= SOL_UVERBS_DRIVER_MAX_MINOR);

        uctxt = uverbs_uobj_get_uctxt_read(id - SOL_UVERBS_DRIVER_MAX_MINOR);
        if (uctxt == NULL) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_write: Failed get user context");
                return (ENXIO);
        }

        if (uctxt->uctxt_type != SOL_UVERBS_UCTXT_VERBS) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_write: write() on invalid uctxt type %x",
                    uctxt->uctxt_type);
                rc = ENXIO;
                goto out;
        }

        if (len < sizeof (hdr)) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_write: Header too small");
                rc =  EINVAL;
                goto out;
        }

        hdr.command     = -1;
        hdr.in_words    = 0;
        hdr.out_words   = 0;

        if (uiomove(&hdr, sizeof (hdr), UIO_WRITE, uiop) != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_write: Error reading header");
                rc = EFAULT;
                goto out;
        }

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "uverbs_write:  hdr.command   = %d", hdr.command);
        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "uverbs_write:  hdr.command   = %d", hdr.command);
        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "uverbs_write:  hdr.in_words  = %d", hdr.in_words);
        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "uverbs_write:  hdr.out_words = %d", hdr.out_words);

        if (hdr.in_words * 4 != len) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_write: Invalid header size");
                rc = EINVAL;
                goto out;
        }

        if (hdr.command >=
            sizeof (uverbs_cmd_table)/sizeof (uverbs_cmd_table[0]) ||
            !uverbs_cmd_table[hdr.command]) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_write: Invalid command (%d)", hdr.command);
                rc = EINVAL;
                goto out;
        }

        ASSERT(len <= SOL_UVERBS_MAX_CMD_PAYLOAD);

        if (uiomove(&payload, len, UIO_WRITE, uiop) != 0) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_write: Error reading payload");
                rc = EFAULT;
                goto out;
        }

#ifdef DEBUG
        unsigned int    *payload_int = (unsigned int *)payload;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "payload:   %08x,    %08x,    %08x,    %08x",
            payload_int[0], payload_int[1],
            payload_int[2], payload_int[3]);
        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "payload:   %08x,    %08x,    %08x,    %08x",
            payload_int[4], payload_int[5],
            payload_int[6], payload_int[7]);
        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "payload:   %08x,    %08x,    %08x,    %08x",
            payload_int[8], payload_int[9],
            payload_int[10], payload_int[11]);
        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "payload:   %08x,    %08x,    %08x",
            payload_int[12], payload_int[13], payload_int[14]);
#endif

        rc = uverbs_cmd_table[hdr.command](uctxt, &payload[0], hdr.in_words * 4,
            hdr.out_words * 4);

out:
        sol_ofs_uobj_put(&uctxt->uobj);

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "uverbs_write: rc = %d", rc);

        return (rc);
}

static int
sol_uverbs_poll(dev_t dev, short events, int anyyet,
    short *reventsp, struct pollhead **phpp)
{
        minor_t                 id = getminor(dev);
        uverbs_uctxt_uobj_t     *uctxt, *verbs_uctxt;
        int                     rc;

#ifdef DEBUG
        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "uverbs_poll(%p, %x, %x, "
            "%p, %p)", dev, events, anyyet, reventsp, phpp);
#endif

        ASSERT(id >= SOL_UVERBS_DRIVER_MAX_MINOR);

        uctxt = uverbs_uobj_get_uctxt_read(id - SOL_UVERBS_DRIVER_MAX_MINOR);
        if (uctxt == NULL) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_poll: Failed get user context");
                return (ENXIO);
        }

        if (uctxt->uctxt_verbs_id < SOL_UVERBS_DRIVER_MAX_MINOR) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_poll: Invalid Verbs user context id, %x",
                    uctxt->uctxt_verbs_id);
                sol_ofs_uobj_put(&uctxt->uobj);
                return (ENXIO);
        }
        verbs_uctxt = uverbs_uobj_get_uctxt_read(uctxt->uctxt_verbs_id
            - SOL_UVERBS_DRIVER_MAX_MINOR);
        if (verbs_uctxt == NULL) {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_poll: Failed get verbs user context");
                sol_ofs_uobj_put(&uctxt->uobj);
                return (ENXIO);
        }
        if (uctxt->uctxt_type == SOL_UVERBS_UCTXT_ASYNC) {
                ASSERT(verbs_uctxt->async_evfile);
                rc = sol_uverbs_event_file_poll(verbs_uctxt->async_evfile,
                    events, anyyet, reventsp, phpp);
        } else if (uctxt->uctxt_type == SOL_UVERBS_UCTXT_COMPL) {
                ASSERT(uctxt->comp_evfile);
                rc = sol_uverbs_event_file_poll(uctxt->comp_evfile,
                    events, anyyet, reventsp, phpp);
        } else {
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "uverbs_poll: poll user context type %d",
                    uctxt->uctxt_type);
                rc = ENXIO;
        }

        sol_ofs_uobj_put(&verbs_uctxt->uobj);
        sol_ofs_uobj_put(&uctxt->uobj);
        return (rc);
}

/*
 * Function:
 *      sol_uverbs_alloc_uctxt
 * Input:
 *      devp     - A pointer to the device number associated with the open.
 *      mod_ctxt - A pointer to the drivers module context.
 *      minor    - The minor device number.
 * Output:
 *      None.
 * Returns:
 *      On success a new user context user resource object associated with
 *      the device passed via devp. NULL on error.
 * Description:
 *      Allocate a new user context user resource object and initialize it.
 *      The users asynchronous event file is created as part of this. On
 *      successful allocation, the user context is returned with the
 *      associated write lock enabled.
 */
static uverbs_uctxt_uobj_t *
sol_uverbs_alloc_uctxt(dev_t *devp, uverbs_module_context_t *mod_ctxt,
    minor_t minor)
{
        uverbs_uctxt_uobj_t *uctxt = NULL;

        uctxt = kmem_zalloc(sizeof (uverbs_uctxt_uobj_t), KM_SLEEP);
        ASSERT(uctxt != NULL);
        sol_ofs_uobj_init(&uctxt->uobj, 0, SOL_UVERBS_UCTXT_UOBJ_TYPE);
        rw_enter(&uctxt->uobj.uo_lock, RW_WRITER);
        if (sol_ofs_uobj_add(&uverbs_uctxt_uo_tbl, &uctxt->uobj) != 0) {
                /*
                 * The initialization routine set's the initial reference,
                 * we dereference the object here to clean it up.
                 */
                SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                    "alloc_uctxt: Object add failed");
                rw_exit(&uctxt->uobj.uo_lock);
                sol_ofs_uobj_free(&uctxt->uobj);
                return (NULL);
        }

        /*
         * Create the new clone for this user context using the
         * object id as the minor number.   Note we offset beyond all
         * real minor device numbers.
         */
        *devp = makedevice(getmajor(*devp),
            uctxt->uobj.uo_id + SOL_UVERBS_DRIVER_MAX_MINOR);

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str, "uverbs_open : "
            "uctxt %p, minor %x- alloced", uctxt,
            uctxt->uobj.uo_id + SOL_UVERBS_DRIVER_MAX_MINOR);

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "alloc_uctxt: user context allocated: %p, ref = %d",
            (void *)uctxt, uctxt->uobj.uo_refcnt);

        mutex_init(&uctxt->lock, NULL, MUTEX_DRIVER, NULL);
        uctxt->mod_ctxt = mod_ctxt;
        if (minor == SOL_UVERBS_DRIVER_EVENT_MINOR) {
                uctxt->uctxt_type = SOL_UVERBS_UCTXT_EVENT;
        } else {
                uctxt->uctxt_type = SOL_UVERBS_UCTXT_VERBS;
                uctxt->hca = &mod_ctxt->hcas[minor];
        }

        init_genlist(&uctxt->pd_list);
        init_genlist(&uctxt->mr_list);
        init_genlist(&uctxt->cq_list);
        init_genlist(&uctxt->srq_list);
        init_genlist(&uctxt->qp_list);
        init_genlist(&uctxt->ah_list);

        /* Return with uobj uo_lock held for WRITTER. */
        return (uctxt);
}

/*
 * Function:
 *      sol_uverbs_qpnum2uqpid
 * Input:
 *      qp_num  - used to find the user object that mapped to this qp_num
 * Output:
 *      None
 * Returns:
 *      DDI_FAILURE if not found else
 *      the uo_id in the user object that matches the qp_num
 * Description:
 *      Find the uo_id of the user object which mapped to the input qp_num
 */
uint32_t
sol_uverbs_qpnum2uqpid(uint32_t qp_num)
{
        sol_ofs_uobj_table_t    *uo_tbl;
        sol_ofs_uobj_t          *uobj;
        uverbs_uqp_uobj_t       *uqp;
        int                     i, j;
        sol_ofs_uobj_blk_t      *blk;

        uo_tbl = &uverbs_uqp_uo_tbl;
        rw_enter(&uo_tbl->uobj_tbl_lock, RW_READER);

        /*
         * Try to find an empty slot for the new user object.
         */
        for (i = 0; i < uo_tbl->uobj_tbl_used_blks; i++) {
                blk = uo_tbl->uobj_tbl_uo_root[i];
                if (blk != NULL) {
                        for (j = 0; j < SOL_OFS_UO_BLKSZ; j++) {
                                if ((uobj = blk->ofs_uoblk_blks[j]) != NULL) {
                                        uqp = (uverbs_uqp_uobj_t *)uobj;
                                        if (uqp->qp_num == qp_num) {
                                                rw_exit(&uo_tbl->uobj_tbl_lock);
                                                SOL_OFS_DPRINTF_L5(
                                                    sol_uverbs_dbg_str,
                                                    "qpnum2uqpid(%x) ret %x",
                                                    qp_num, uobj->uo_id);
                                                return (uobj->uo_id);
                                        }
                                }
                        }
                }
        }

        rw_exit(&uo_tbl->uobj_tbl_lock);
        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str, "qpnum2uqpid(%x) ret %x",
            qp_num, DDI_FAILURE);
        return (DDI_FAILURE);
}

void
sol_uverbs_get_clnt_hdl(void **ibclnt_hdl, void **iwclnt_hdl)
{
        *ibclnt_hdl = sol_uverbs_ib_clntp;
        *iwclnt_hdl = NULL;
}

void *
sol_uverbs_qpnum2qphdl(uint32_t qpnum)
{
        int32_t uqpid;

        uqpid = sol_uverbs_qpnum2uqpid(qpnum);
        if (uqpid == DDI_FAILURE)
                return (NULL);
        return (sol_uverbs_uqpid_to_ibt_handle(uqpid));
}

int
sol_uverbs_disable_uqpn_modify(uint32_t qpnum)
{
        int32_t uqpid;

        uqpid = sol_uverbs_qpnum2uqpid(qpnum);
        if (uqpid == DDI_FAILURE)
                return (-1);

        return (sol_uverbs_disable_user_qp_modify(uqpid));
}

extern int uverbs_uqpn_cq_ctrl(uint32_t, sol_uverbs_cq_ctrl_t);

int
sol_uverbs_uqpn_cq_ctrl(uint32_t qpnum, sol_uverbs_cq_ctrl_t ctrl)
{
        int32_t uqpid;

        uqpid = sol_uverbs_qpnum2uqpid(qpnum);
        if (uqpid == DDI_FAILURE)
                return (-1);

        return (uverbs_uqpn_cq_ctrl(uqpid, ctrl));
}

void
sol_uverbs_set_qp_free_state(sol_uverbs_qp_free_state_t qp_free_state,
    uint32_t qpnum, void *qphdl)
{
        int32_t                 uqpid;
        uverbs_uqp_uobj_t       *uqp;

        SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
            "sol_uverbs_set_qp_free_state(%x, %x, %p)",
            qp_free_state, qpnum, qphdl);
        if (qp_free_state == SOL_UVERBS2UCMA_DISABLE_QP_FREE) {
                uqpid = sol_uverbs_qpnum2uqpid(qpnum);
                if (uqpid == DDI_FAILURE) {
                        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                            "set_qp_free_state(%d)-invalid qpnum",
                            qpnum);
                        return;
                }

                uqp = uverbs_uobj_get_uqp_write(uqpid);
                if (uqp == NULL) {
                        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                            "set_qp_free_state(%d)-uqp lookup failure", qpnum);
                        return;
                }
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "set_qp_free_state : uqp %p, setting Disable QP Free", uqp);
                uqp->uqp_free_state  = SOL_UVERBS2UCMA_DISABLE_QP_FREE;
                sol_ofs_uobj_put(&uqp->uobj);
                return;
        }

        ASSERT(qphdl);
        uqp = (uverbs_uqp_uobj_t *)ibt_get_qp_private((ibt_qp_hdl_t)qphdl);
        ASSERT(uqp);
        if (uqp->uqp_free_state != SOL_UVERBS2UCMA_FREE_PENDING) {
                /*
                 * Enable free flag, so that close or userland free_qp
                 * call can free this in the future.
                 */
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "set_qp_free_state : uqp %p, setting Enable QP Free",
                    uqp);
                rw_enter(&(uqp->uobj.uo_lock), RW_WRITER);
                uqp->uqp_free_state = SOL_UVERBS2UCMA_ENABLE_QP_FREE;
                rw_exit(&(uqp->uobj.uo_lock));
        } else {
                /*
                 * uqp_free_state is set to FREE_PENDING, QP has been freed
                 * by userland. Call uverbs_uqp_free() to free this.
                 */
                SOL_OFS_DPRINTF_L5(sol_uverbs_dbg_str,
                    "set_qp_free_state : uqp %p calling uverbs_uqp_free()",
                    uqp);
                rw_enter(&(uqp->uobj.uo_lock), RW_WRITER);
                sol_ofs_uobj_ref(&uqp->uobj);
                if (uverbs_uqp_free(uqp, uqp->uctxt))
                        SOL_OFS_DPRINTF_L2(sol_uverbs_dbg_str,
                            "set_qp_free_state : uverbs_uqp_free(%p) failed",
                            uqp);
        }
}

/*
 * Function:
 *      sol_uverbs_user_objects_init
 * Input:
 *      None
 * Output:
 *      None
 * Returns:
 *      None
 * Description:
 *      Initializes all of the user object resource managment tables.
 */
static void sol_uverbs_user_objects_init()
{
        sol_ofs_uobj_tbl_init(&uverbs_uctxt_uo_tbl,
            sizeof (uverbs_uctxt_uobj_t));
        sol_ofs_uobj_tbl_init(&uverbs_upd_uo_tbl,
            sizeof (uverbs_upd_uobj_t));
        sol_ofs_uobj_tbl_init(&uverbs_umr_uo_tbl,
            sizeof (uverbs_umr_uobj_t));
        sol_ofs_uobj_tbl_init(&uverbs_ucq_uo_tbl,
            sizeof (uverbs_ucq_uobj_t));
        sol_ofs_uobj_tbl_init(&uverbs_usrq_uo_tbl,
            sizeof (uverbs_usrq_uobj_t));
        sol_ofs_uobj_tbl_init(&uverbs_uqp_uo_tbl,
            sizeof (uverbs_uqp_uobj_t));
        sol_ofs_uobj_tbl_init(&uverbs_uah_uo_tbl,
            sizeof (uverbs_uah_uobj_t));
        sol_ofs_uobj_tbl_init(&uverbs_ufile_uo_tbl,
            sizeof (uverbs_ufile_uobj_t));
}

/*
 * Function:
 *      sol_uverbs_user_objects_fini
 * Input:
 *      None
 * Output:
 *      None
 * Returns:
 *      None
 * Description:
 *      Releases all of the user object resource managment tables.
 */
static void sol_uverbs_user_objects_fini()
{
        sol_ofs_uobj_tbl_fini(&uverbs_ufile_uo_tbl);
        sol_ofs_uobj_tbl_fini(&uverbs_uah_uo_tbl);
        sol_ofs_uobj_tbl_fini(&uverbs_uqp_uo_tbl);
        sol_ofs_uobj_tbl_fini(&uverbs_usrq_uo_tbl);
        sol_ofs_uobj_tbl_fini(&uverbs_ucq_uo_tbl);
        sol_ofs_uobj_tbl_fini(&uverbs_umr_uo_tbl);
        sol_ofs_uobj_tbl_fini(&uverbs_upd_uo_tbl);
        sol_ofs_uobj_tbl_fini(&uverbs_uctxt_uo_tbl);
}

/*
 * Function:
 *      sol_uverbs_ibt_to_kernel_status
 * Input:
 *      status  - An IBT status code.
 * Output:
 *      None
 * Returns:
 *      The "errno" based kernel error code the IBT status maps to.
 * Description:
 *      Map an IBT status to the "errno" code that should be returned.
 */
int
sol_uverbs_ibt_to_kernel_status(ibt_status_t status)
{
        int err;

        switch (status) {
                case IBT_NOT_SUPPORTED:
                        err = ENOTSUP;
                        break;

                case IBT_ILLEGAL_OP:
                case IBT_INVALID_PARAM:
                        err = EINVAL;
                        break;

                case IBT_HCA_IN_USE:
                case IBT_HCA_BUSY_DETACHING:
                case IBT_HCA_BUSY_CLOSING:
                case IBT_CHAN_IN_USE:
                case IBT_CQ_BUSY:
                case IBT_MR_IN_USE:
                case IBT_PD_IN_USE:
                case IBT_SRQ_IN_USE:
                        err = EBUSY;
                        break;
                case    IBT_INSUFF_RESOURCE:
                case    IBT_INSUFF_KERNEL_RESOURCE:
                case    IBT_HCA_WR_EXCEEDED:
                case    IBT_HCA_SGL_EXCEEDED:
                        err = ENOMEM;
                        break;

                default:
                        err = EINVAL;
        }
        return (err);
}

/* ARGSUSED */
uint32_t
sol_uverbs_ibt_to_of_device_cap_flags(ibt_hca_flags_t flags,
    ibt_hca_flags2_t flags2) {

        uint32_t of_flags = 0;

        if (flags && IBT_HCA_RESIZE_CHAN)
                of_flags |= IB_DEVICE_RESIZE_MAX_WR;

        if (flags && IBT_HCA_PKEY_CNTR)
                of_flags |= IB_DEVICE_BAD_PKEY_CNTR;

        if (flags && IBT_HCA_QKEY_CNTR)
                of_flags |= IB_DEVICE_BAD_QKEY_CNTR;

        if (flags && IBT_HCA_RAW_MULTICAST)
                of_flags |= IB_DEVICE_RAW_MULTI;

        if (flags && IBT_HCA_AUTO_PATH_MIG)
                of_flags |= IB_DEVICE_AUTO_PATH_MIG;

        if (flags && IBT_HCA_SQD_SQD_PORT)
                of_flags |= IB_DEVICE_CHANGE_PHY_PORT;

        if (flags && IBT_HCA_AH_PORT_CHECK)
                of_flags |= IB_DEVICE_UD_AV_PORT_ENFORCE;

        if (flags && IBT_HCA_CURRENT_QP_STATE)
                of_flags |= IB_DEVICE_CURR_QP_STATE_MOD;

        if (flags && IBT_HCA_SHUTDOWN_PORT)
                of_flags |= IB_DEVICE_SHUTDOWN_PORT;

        if (flags && IBT_HCA_INIT_TYPE)
                of_flags |= IB_DEVICE_INIT_TYPE;

        if (flags && IBT_HCA_PORT_UP)
                of_flags |= IB_DEVICE_PORT_ACTIVE_EVENT;

        if (flags && IBT_HCA_SI_GUID)
                of_flags |= IB_DEVICE_SYS_IMAGE_GUID;

        if (flags && IBT_HCA_RNR_NAK)
                of_flags |= IB_DEVICE_RC_RNR_NAK_GEN;

        if (flags && IBT_HCA_RESIZE_SRQ)
                of_flags |= IB_DEVICE_SRQ_RESIZE;

        if (flags && IBT_HCA_BASE_QUEUE_MGT)
                of_flags |= IB_DEVICE_N_NOTIFY_CQ;

        if (flags && IBT_HCA_ZERO_BASED_VA)
                of_flags |= IB_DEVICE_ZERO_STAG;

        if (flags && IBT_HCA_LOCAL_INVAL_FENCE)
                of_flags |= IB_DEVICE_SEND_W_INV;

        if (flags && IBT_HCA_MEM_WIN_TYPE_2B)
                of_flags |= IB_DEVICE_MEM_WINDOW;

        return (of_flags);
}

uint64_t
sol_uverbs_ibt_to_of_page_sz(ibt_page_sizes_t page_szs)
{

        uint64_t of_page_sz = 0;

        if (page_szs && IBT_PAGE_4K)
                of_page_sz |= 1LL << 12;

        if (page_szs && IBT_PAGE_8K)
                of_page_sz |= 1LL << 13;

        if (page_szs && IBT_PAGE_16K)
                of_page_sz |= 1LL << 14;

        if (page_szs && IBT_PAGE_32K)
                of_page_sz |= 1LL << 15;

        if (page_szs && IBT_PAGE_64K)
                of_page_sz |= 1LL << 16;

        if (page_szs && IBT_PAGE_128K)
                of_page_sz |= 1LL << 17;

        if (page_szs && IBT_PAGE_256K)
                of_page_sz |= 1LL << 18;

        if (page_szs && IBT_PAGE_512K)
                of_page_sz |= 1LL << 19;

        if (page_szs && IBT_PAGE_1M)
                of_page_sz |= 1LL << 20;

        if (page_szs && IBT_PAGE_2M)
                of_page_sz |= 1LL << 21;

        if (page_szs && IBT_PAGE_4M)
                of_page_sz |= 1LL << 22;

        if (page_szs && IBT_PAGE_8M)
                of_page_sz |= 1LL << 23;

        if (page_szs && IBT_PAGE_16M)
                of_page_sz |= 1LL << 24;

        if (page_szs && IBT_PAGE_32M)
                of_page_sz |= 1LL << 25;

        if (page_szs && IBT_PAGE_64M)
                of_page_sz |= 1LL << 26;

        if (page_szs && IBT_PAGE_128M)
                of_page_sz |= 1LL << 27;

        if (page_szs && IBT_PAGE_256M)
                of_page_sz |= 1LL << 28;

        if (page_szs && IBT_PAGE_512M)
                of_page_sz |= 1LL << 29;

        if (page_szs && IBT_PAGE_1G)
                of_page_sz |= 1LL << 30;

        if (page_szs && IBT_PAGE_2G)
                of_page_sz |= 1LL << 31;

        if (page_szs && IBT_PAGE_4G)
                of_page_sz |= 1LL << 32;

        if (page_szs && IBT_PAGE_8G)
                of_page_sz |= 1LL << 33;

        if (page_szs && IBT_PAGE_16G)
                of_page_sz |= 1LL << 34;

        return (of_page_sz);
}