/*
 * CDDL HEADER START
 *
 * The contents of this file are subject to the terms of the
 * Common Development and Distribution License (the "License").
 * You may not use this file except in compliance with the License.
 *
 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
 * or http://www.opensolaris.org/os/licensing.
 * See the License for the specific language governing permissions
 * and limitations under the License.
 *
 * When distributing Covered Code, include this CDDL HEADER in each
 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
 * If applicable, add the following below this CDDL HEADER, with the
 * fields enclosed by brackets "[]" replaced with your own identifying
 * information: Portions Copyright [yyyy] [name of copyright owner]
 *
 * CDDL HEADER END
 */
/*
 * Copyright 2010 Sun Microsystems, Inc.  All rights reserved.
 * Use is subject to license terms.
 */
/* Copyright (c) 1983, 1984, 1985, 1986, 1987, 1988, 1989 AT&T */
/* All Rights Reserved */
/*
 * Portions of this source code were derived from Berkeley
 * 4.3 BSD under license from the Regents of the University of
 * California.
 */

#include <sys/param.h>
#include <sys/types.h>
#include <sys/user.h>
#include <sys/systm.h>
#include <sys/sysmacros.h>
#include <sys/errno.h>
#include <sys/kmem.h>
#include <sys/debug.h>
#include <sys/systm.h>
#include <sys/kstat.h>
#include <sys/t_lock.h>
#include <sys/ddi.h>
#include <sys/cmn_err.h>
#include <sys/time.h>
#include <sys/isa_defs.h>
#include <sys/zone.h>
#include <sys/sdt.h>

#include <rpc/types.h>
#include <rpc/xdr.h>
#include <rpc/auth.h>
#include <rpc/clnt.h>
#include <rpc/rpc_msg.h>
#include <rpc/rpc_rdma.h>
#include <nfs/nfs.h>
#include <nfs/nfs4_kprot.h>

static uint32_t rdma_bufs_rqst = RDMA_BUFS_RQST;

static int clnt_compose_rpcmsg(CLIENT *, rpcproc_t, rdma_buf_t *,
                            XDR *, xdrproc_t, caddr_t);
static int  clnt_compose_rdma_header(CONN *, CLIENT *, rdma_buf_t *,
                    XDR **, uint_t *);
static int clnt_setup_rlist(CONN *, XDR *, XDR *);
static int clnt_setup_wlist(CONN *, XDR *, XDR *, rdma_buf_t *);
static int clnt_setup_long_reply(CONN *, struct clist **, uint_t);
static void clnt_check_credit(CONN *);
static void clnt_return_credit(CONN *);
static void clnt_decode_long_reply(CONN *, struct clist *,
                struct clist *, XDR *, XDR **, struct clist *,
                struct clist *, uint_t, uint_t);

static void clnt_update_credit(CONN *, uint32_t);

static enum clnt_stat clnt_rdma_kcallit(CLIENT *, rpcproc_t, xdrproc_t,
    caddr_t, xdrproc_t, caddr_t, struct timeval);
static void     clnt_rdma_kabort(CLIENT *);
static void     clnt_rdma_kerror(CLIENT *, struct rpc_err *);
static bool_t   clnt_rdma_kfreeres(CLIENT *, xdrproc_t, caddr_t);
static void     clnt_rdma_kdestroy(CLIENT *);
static bool_t   clnt_rdma_kcontrol(CLIENT *, int, char *);
static int      clnt_rdma_ksettimers(CLIENT *, struct rpc_timers *,
    struct rpc_timers *, int, void(*)(int, int, caddr_t), caddr_t, uint32_t);

/*
 * Operations vector for RDMA based RPC
 */
static struct clnt_ops rdma_clnt_ops = {
        clnt_rdma_kcallit,      /* do rpc call */
        clnt_rdma_kabort,       /* abort call */
        clnt_rdma_kerror,       /* return error status */
        clnt_rdma_kfreeres,     /* free results */
        clnt_rdma_kdestroy,     /* destroy rpc handle */
        clnt_rdma_kcontrol,     /* the ioctl() of rpc */
        clnt_rdma_ksettimers,   /* set retry timers */
};

/*
 * The size of the preserialized RPC header information.
 */
#define CKU_HDRSIZE     20
#define CLNT_RDMA_SUCCESS 0
#define CLNT_RDMA_FAIL (-1)

#define AUTH_REFRESH_COUNT 2

#define IS_RPCSEC_GSS(authh)                    \
        (authh->cl_auth->ah_cred.oa_flavor == RPCSEC_GSS)

/*
 * Per RPC RDMA endpoint details
 */
typedef struct cku_private {
        CLIENT                  cku_client;     /* client handle */
        rdma_mod_t              *cku_rd_mod;    /* underlying RDMA mod */
        void                    *cku_rd_handle; /* underlying RDMA device */
        struct netbuf           cku_srcaddr;    /* source address for retries */
        struct netbuf           cku_addr;       /* remote netbuf address */
        int                     cku_addrfmly;   /* for finding addr_type */
        struct rpc_err          cku_err;        /* error status */
        struct cred             *cku_cred;      /* credentials */
        XDR                     cku_outxdr;     /* xdr stream for output */
        uint32_t                cku_outsz;
        XDR                     cku_inxdr;      /* xdr stream for input */
        char                    cku_rpchdr[CKU_HDRSIZE+4]; /* rpc header */
        uint32_t                cku_xid;        /* current XID */
} cku_private_t;

#define CLNT_RDMA_DELAY 10      /* secs to delay after a connection failure */
static int clnt_rdma_min_delay = CLNT_RDMA_DELAY;

struct {
        kstat_named_t   rccalls;
        kstat_named_t   rcbadcalls;
        kstat_named_t   rcbadxids;
        kstat_named_t   rctimeouts;
        kstat_named_t   rcnewcreds;
        kstat_named_t   rcbadverfs;
        kstat_named_t   rctimers;
        kstat_named_t   rccantconn;
        kstat_named_t   rcnomem;
        kstat_named_t   rcintrs;
        kstat_named_t   rclongrpcs;
} rdmarcstat = {
        { "calls",      KSTAT_DATA_UINT64 },
        { "badcalls",   KSTAT_DATA_UINT64 },
        { "badxids",    KSTAT_DATA_UINT64 },
        { "timeouts",   KSTAT_DATA_UINT64 },
        { "newcreds",   KSTAT_DATA_UINT64 },
        { "badverfs",   KSTAT_DATA_UINT64 },
        { "timers",     KSTAT_DATA_UINT64 },
        { "cantconn",   KSTAT_DATA_UINT64 },
        { "nomem",      KSTAT_DATA_UINT64 },
        { "interrupts", KSTAT_DATA_UINT64 },
        { "longrpc",    KSTAT_DATA_UINT64 }
};

kstat_named_t *rdmarcstat_ptr = (kstat_named_t *)&rdmarcstat;
uint_t rdmarcstat_ndata = sizeof (rdmarcstat) / sizeof (kstat_named_t);

#ifdef DEBUG
int rdma_clnt_debug = 0;
#endif

#ifdef accurate_stats
extern kmutex_t rdmarcstat_lock;    /* mutex for rcstat updates */

#define RCSTAT_INCR(x)                  \
        mutex_enter(&rdmarcstat_lock);  \
        rdmarcstat.x.value.ui64++;      \
        mutex_exit(&rdmarcstat_lock);
#else
#define RCSTAT_INCR(x)                  \
        rdmarcstat.x.value.ui64++;
#endif

#define ptoh(p)         (&((p)->cku_client))
#define htop(h)         ((cku_private_t *)((h)->cl_private))

uint_t
calc_length(uint_t len)
{
        len = RNDUP(len);

        if (len <= 64 * 1024) {
                if (len > 32 * 1024) {
                        len = 64 * 1024;
                } else {
                        if (len > 16 * 1024) {
                                len = 32 * 1024;
                        } else {
                                if (len > 8 * 1024) {
                                        len = 16 * 1024;
                                } else {
                                        len = 8 * 1024;
                                }
                        }
                }
        }
        return (len);
}
int
clnt_rdma_kcreate(char *proto, void *handle, struct netbuf *raddr, int family,
    rpcprog_t pgm, rpcvers_t vers, struct cred *cred, CLIENT **cl)
{
        CLIENT *h;
        struct cku_private *p;
        struct rpc_msg call_msg;
        rdma_registry_t *rp;

        ASSERT(INGLOBALZONE(curproc));

        if (cl == NULL)
                return (EINVAL);
        *cl = NULL;

        p = kmem_zalloc(sizeof (*p), KM_SLEEP);

        /*
         * Find underlying RDMATF plugin
         */
        rw_enter(&rdma_lock, RW_READER);
        rp = rdma_mod_head;
        while (rp != NULL) {
                if (strcmp(rp->r_mod->rdma_api, proto))
                        rp = rp->r_next;
                else {
                        p->cku_rd_mod = rp->r_mod;
                        p->cku_rd_handle = handle;
                        break;
                }
        }
        rw_exit(&rdma_lock);

        if (p->cku_rd_mod == NULL) {
                /*
                 * Should not happen.
                 * No matching RDMATF plugin.
                 */
                kmem_free(p, sizeof (struct cku_private));
                return (EINVAL);
        }

        h = ptoh(p);
        h->cl_ops = &rdma_clnt_ops;
        h->cl_private = (caddr_t)p;
        h->cl_auth = authkern_create();

        /* call message, just used to pre-serialize below */
        call_msg.rm_xid = 0;
        call_msg.rm_direction = CALL;
        call_msg.rm_call.cb_rpcvers = RPC_MSG_VERSION;
        call_msg.rm_call.cb_prog = pgm;
        call_msg.rm_call.cb_vers = vers;

        xdrmem_create(&p->cku_outxdr, p->cku_rpchdr, CKU_HDRSIZE, XDR_ENCODE);
        /* pre-serialize call message header */
        if (!xdr_callhdr(&p->cku_outxdr, &call_msg)) {
                XDR_DESTROY(&p->cku_outxdr);
                auth_destroy(h->cl_auth);
                kmem_free(p, sizeof (struct cku_private));
                return (EINVAL);
        }

        /*
         * Set up the rpc information
         */
        p->cku_cred = cred;
        p->cku_srcaddr.buf = kmem_zalloc(raddr->maxlen, KM_SLEEP);
        p->cku_srcaddr.maxlen = raddr->maxlen;
        p->cku_srcaddr.len = 0;
        p->cku_addr.buf = kmem_zalloc(raddr->maxlen, KM_SLEEP);
        p->cku_addr.maxlen = raddr->maxlen;
        p->cku_addr.len = raddr->len;
        bcopy(raddr->buf, p->cku_addr.buf, raddr->len);
        p->cku_addrfmly = family;

        *cl = h;
        return (0);
}

static void
clnt_rdma_kdestroy(CLIENT *h)
{
        struct cku_private *p = htop(h);

        kmem_free(p->cku_srcaddr.buf, p->cku_srcaddr.maxlen);
        kmem_free(p->cku_addr.buf, p->cku_addr.maxlen);
        kmem_free(p, sizeof (*p));
}

void
clnt_rdma_kinit(CLIENT *h, char *proto, void *handle, struct netbuf *raddr,
    struct cred *cred)
{
        struct cku_private *p = htop(h);
        rdma_registry_t *rp;

        ASSERT(INGLOBALZONE(curproc));
        /*
         * Find underlying RDMATF plugin
         */
        p->cku_rd_mod = NULL;
        rw_enter(&rdma_lock, RW_READER);
        rp = rdma_mod_head;
        while (rp != NULL) {
                if (strcmp(rp->r_mod->rdma_api, proto))
                        rp = rp->r_next;
                else {
                        p->cku_rd_mod = rp->r_mod;
                        p->cku_rd_handle = handle;
                        break;
                }

        }
        rw_exit(&rdma_lock);

        /*
         * Set up the rpc information
         */
        p->cku_cred = cred;
        p->cku_xid = 0;

        if (p->cku_addr.maxlen < raddr->len) {
                if (p->cku_addr.maxlen != 0 && p->cku_addr.buf != NULL)
                        kmem_free(p->cku_addr.buf, p->cku_addr.maxlen);
                p->cku_addr.buf = kmem_zalloc(raddr->maxlen, KM_SLEEP);
                p->cku_addr.maxlen = raddr->maxlen;
        }

        p->cku_srcaddr.len = 0;

        p->cku_addr.len = raddr->len;
        bcopy(raddr->buf, p->cku_addr.buf, raddr->len);
        h->cl_ops = &rdma_clnt_ops;
}

static int
clnt_compose_rpcmsg(CLIENT *h, rpcproc_t procnum,
    rdma_buf_t *rpcmsg, XDR *xdrs,
    xdrproc_t xdr_args, caddr_t argsp)
{
        cku_private_t *p = htop(h);

        if (h->cl_auth->ah_cred.oa_flavor != RPCSEC_GSS) {
                /*
                 * Copy in the preserialized RPC header
                 * information.
                 */
                bcopy(p->cku_rpchdr, rpcmsg->addr, CKU_HDRSIZE);

                /*
                 * transaction id is the 1st thing in the output
                 * buffer.
                 */
                /* LINTED pointer alignment */
                (*(uint32_t *)(rpcmsg->addr)) = p->cku_xid;

                /* Skip the preserialized stuff. */
                XDR_SETPOS(xdrs, CKU_HDRSIZE);

                /* Serialize dynamic stuff into the output buffer. */
                if ((!XDR_PUTINT32(xdrs, (int32_t *)&procnum)) ||
                    (!AUTH_MARSHALL(h->cl_auth, xdrs, p->cku_cred)) ||
                    (!(*xdr_args)(xdrs, argsp))) {
                        DTRACE_PROBE(krpc__e__clntrdma__rpcmsg__dynargs);
                        return (CLNT_RDMA_FAIL);
                }
                p->cku_outsz = XDR_GETPOS(xdrs);
        } else {
                uint32_t *uproc = (uint32_t *)&p->cku_rpchdr[CKU_HDRSIZE];
                IXDR_PUT_U_INT32(uproc, procnum);
                (*(uint32_t *)(&p->cku_rpchdr[0])) = p->cku_xid;
                XDR_SETPOS(xdrs, 0);

                /* Serialize the procedure number and the arguments. */
                if (!AUTH_WRAP(h->cl_auth, (caddr_t)p->cku_rpchdr,
                    CKU_HDRSIZE+4, xdrs, xdr_args, argsp)) {
                        if (rpcmsg->addr != xdrs->x_base) {
                                rpcmsg->addr = xdrs->x_base;
                                rpcmsg->len = xdr_getbufsize(xdrs);
                        }
                        DTRACE_PROBE(krpc__e__clntrdma__rpcmsg__procnum);
                        return (CLNT_RDMA_FAIL);
                }
                /*
                 * If we had to allocate a new buffer while encoding
                 * then update the addr and len.
                 */
                if (rpcmsg->addr != xdrs->x_base) {
                        rpcmsg->addr = xdrs->x_base;
                        rpcmsg->len = xdr_getbufsize(xdrs);
                }

                p->cku_outsz = XDR_GETPOS(xdrs);
                DTRACE_PROBE1(krpc__i__compose__size__sec, int, p->cku_outsz)
        }

        return (CLNT_RDMA_SUCCESS);
}

static int
clnt_compose_rdma_header(CONN *conn, CLIENT *h, rdma_buf_t *clmsg,
    XDR **xdrs, uint_t *op)
{
        cku_private_t *p = htop(h);
        uint_t vers;
        uint32_t rdma_credit = rdma_bufs_rqst;

        vers = RPCRDMA_VERS;
        clmsg->type = SEND_BUFFER;

        if (rdma_buf_alloc(conn, clmsg)) {
                return (CLNT_RDMA_FAIL);
        }

        *xdrs = &p->cku_outxdr;
        xdrmem_create(*xdrs, clmsg->addr, clmsg->len, XDR_ENCODE);

        (*(uint32_t *)clmsg->addr) = p->cku_xid;
        XDR_SETPOS(*xdrs, sizeof (uint32_t));
        (void) xdr_u_int(*xdrs, &vers);
        (void) xdr_u_int(*xdrs, &rdma_credit);
        (void) xdr_u_int(*xdrs, op);

        return (CLNT_RDMA_SUCCESS);
}

/*
 * If xp_cl is NULL value, then the RPC payload will NOT carry
 * an RDMA READ chunk list, in this case we insert FALSE into
 * the XDR stream. Otherwise we use the clist and RDMA register
 * the memory and encode the clist into the outbound XDR stream.
 */
static int
clnt_setup_rlist(CONN *conn, XDR *xdrs, XDR *call_xdrp)
{
        int status;
        struct clist *rclp;
        int32_t xdr_flag = XDR_RDMA_RLIST_REG;

        XDR_CONTROL(call_xdrp, XDR_RDMA_GET_RLIST, &rclp);

        if (rclp != NULL) {
                status = clist_register(conn, rclp, CLIST_REG_SOURCE);
                if (status != RDMA_SUCCESS) {
                        return (CLNT_RDMA_FAIL);
                }
                XDR_CONTROL(call_xdrp, XDR_RDMA_SET_FLAGS, &xdr_flag);
        }
        (void) xdr_do_clist(xdrs, &rclp);

        return (CLNT_RDMA_SUCCESS);
}

/*
 * If xp_wcl is NULL value, then the RPC payload will NOT carry
 * an RDMA WRITE chunk list, in this case we insert FALSE into
 * the XDR stream. Otherwise we use the clist and  RDMA register
 * the memory and encode the clist into the outbound XDR stream.
 */
static int
clnt_setup_wlist(CONN *conn, XDR *xdrs, XDR *call_xdrp, rdma_buf_t *rndbuf)
{
        int status;
        struct clist *wlist, *rndcl;
        int wlen, rndlen;
        int32_t xdr_flag = XDR_RDMA_WLIST_REG;

        XDR_CONTROL(call_xdrp, XDR_RDMA_GET_WLIST, &wlist);

        if (wlist != NULL) {
                /*
                 * If we are sending a non 4-byte alligned length
                 * the server will roundup the length to 4-byte
                 * boundary. In such a case, a trailing chunk is
                 * added to take any spill over roundup bytes.
                 */
                wlen = clist_len(wlist);
                rndlen = (roundup(wlen, BYTES_PER_XDR_UNIT) - wlen);
                if (rndlen) {
                        rndcl = clist_alloc();
                        /*
                         * calc_length() will allocate a PAGESIZE
                         * buffer below.
                         */
                        rndcl->c_len = calc_length(rndlen);
                        rndcl->rb_longbuf.type = RDMA_LONG_BUFFER;
                        rndcl->rb_longbuf.len = rndcl->c_len;
                        if (rdma_buf_alloc(conn, &rndcl->rb_longbuf)) {
                                clist_free(rndcl);
                                return (CLNT_RDMA_FAIL);
                        }

                        /* Roundup buffer freed back in caller */
                        *rndbuf = rndcl->rb_longbuf;

                        rndcl->u.c_daddr3 = rndcl->rb_longbuf.addr;
                        rndcl->c_next = NULL;
                        rndcl->c_dmemhandle = rndcl->rb_longbuf.handle;
                        wlist->c_next = rndcl;
                }

                status = clist_register(conn, wlist, CLIST_REG_DST);
                if (status != RDMA_SUCCESS) {
                        rdma_buf_free(conn, rndbuf);
                        bzero(rndbuf, sizeof (rdma_buf_t));
                        return (CLNT_RDMA_FAIL);
                }
                XDR_CONTROL(call_xdrp, XDR_RDMA_SET_FLAGS, &xdr_flag);
        }

        if (!xdr_encode_wlist(xdrs, wlist)) {
                if (rndlen) {
                        rdma_buf_free(conn, rndbuf);
                        bzero(rndbuf, sizeof (rdma_buf_t));
                }
                return (CLNT_RDMA_FAIL);
        }

        return (CLNT_RDMA_SUCCESS);
}

static int
clnt_setup_long_reply(CONN *conn, struct clist **clpp, uint_t length)
{
        if (length == 0) {
                *clpp = NULL;
                return (CLNT_RDMA_SUCCESS);
        }

        *clpp = clist_alloc();

        (*clpp)->rb_longbuf.len = calc_length(length);
        (*clpp)->rb_longbuf.type = RDMA_LONG_BUFFER;

        if (rdma_buf_alloc(conn, &((*clpp)->rb_longbuf))) {
                clist_free(*clpp);
                *clpp = NULL;
                return (CLNT_RDMA_FAIL);
        }

        (*clpp)->u.c_daddr3 = (*clpp)->rb_longbuf.addr;
        (*clpp)->c_len = (*clpp)->rb_longbuf.len;
        (*clpp)->c_next = NULL;
        (*clpp)->c_dmemhandle = (*clpp)->rb_longbuf.handle;

        if (clist_register(conn, *clpp, CLIST_REG_DST)) {
                DTRACE_PROBE(krpc__e__clntrdma__longrep_regbuf);
                rdma_buf_free(conn, &((*clpp)->rb_longbuf));
                clist_free(*clpp);
                *clpp = NULL;
                return (CLNT_RDMA_FAIL);
        }

        return (CLNT_RDMA_SUCCESS);
}

/* ARGSUSED */
static enum clnt_stat
clnt_rdma_kcallit(CLIENT *h, rpcproc_t procnum, xdrproc_t xdr_args,
    caddr_t argsp, xdrproc_t xdr_results, caddr_t resultsp,
    struct timeval wait)
{
        cku_private_t *p = htop(h);

        int     try_call_again;
        int     refresh_attempt = AUTH_REFRESH_COUNT;
        int     status;
        int     msglen;

        XDR     *call_xdrp, callxdr; /* for xdrrdma encoding the RPC call */
        XDR     *reply_xdrp, replyxdr; /* for xdrrdma decoding the RPC reply */
        XDR     *rdmahdr_o_xdrs, *rdmahdr_i_xdrs;

        struct rpc_msg  reply_msg;
        rdma_registry_t *m;

        struct clist *cl_sendlist;
        struct clist *cl_recvlist;
        struct clist *cl;
        struct clist *cl_rpcmsg;
        struct clist *cl_rdma_reply;
        struct clist *cl_rpcreply_wlist;
        struct clist *cl_long_reply;
        rdma_buf_t  rndup;

        uint_t vers;
        uint_t op;
        uint_t off;
        uint32_t seg_array_len;
        uint_t long_reply_len;
        uint_t rpcsec_gss;
        uint_t gss_i_or_p;

        CONN *conn = NULL;
        rdma_buf_t clmsg;
        rdma_buf_t rpcmsg;
        rdma_chunkinfo_lengths_t rcil;

        clock_t ticks;
        bool_t wlist_exists_reply;

        uint32_t rdma_credit = rdma_bufs_rqst;

        RCSTAT_INCR(rccalls);

call_again:

        bzero(&clmsg, sizeof (clmsg));
        bzero(&rpcmsg, sizeof (rpcmsg));
        bzero(&rndup, sizeof (rndup));
        try_call_again = 0;
        cl_sendlist = NULL;
        cl_recvlist = NULL;
        cl = NULL;
        cl_rpcmsg = NULL;
        cl_rdma_reply = NULL;
        call_xdrp = NULL;
        reply_xdrp = NULL;
        wlist_exists_reply  = FALSE;
        cl_rpcreply_wlist = NULL;
        cl_long_reply = NULL;
        rcil.rcil_len = 0;
        rcil.rcil_len_alt = 0;
        long_reply_len = 0;

        rw_enter(&rdma_lock, RW_READER);
        m = (rdma_registry_t *)p->cku_rd_handle;
        if (m->r_mod_state == RDMA_MOD_INACTIVE) {
                /*
                 * If we didn't find a matching RDMA module in the registry
                 * then there is no transport.
                 */
                rw_exit(&rdma_lock);
                p->cku_err.re_status = RPC_CANTSEND;
                p->cku_err.re_errno = EIO;
                ticks = clnt_rdma_min_delay * drv_usectohz(1000000);
                if (h->cl_nosignal == TRUE) {
                        delay(ticks);
                } else {
                        if (delay_sig(ticks) == EINTR) {
                                p->cku_err.re_status = RPC_INTR;
                                p->cku_err.re_errno = EINTR;
                        }
                }
                return (RPC_CANTSEND);
        }
        /*
         * Get unique xid
         */
        if (p->cku_xid == 0)
                p->cku_xid = alloc_xid();

        status = RDMA_GET_CONN(p->cku_rd_mod->rdma_ops, &p->cku_srcaddr,
            &p->cku_addr, p->cku_addrfmly, p->cku_rd_handle, &conn);
        rw_exit(&rdma_lock);

        /*
         * If there is a problem with the connection reflect the issue
         * back to the higher level to address, we MAY delay for a short
         * period so that we are kind to the transport.
         */
        if (conn == NULL) {
                /*
                 * Connect failed to server. Could be because of one
                 * of several things. In some cases we don't want
                 * the caller to retry immediately - delay before
                 * returning to caller.
                 */
                switch (status) {
                case RDMA_TIMEDOUT:
                        /*
                         * Already timed out. No need to delay
                         * some more.
                         */
                        p->cku_err.re_status = RPC_TIMEDOUT;
                        p->cku_err.re_errno = ETIMEDOUT;
                        break;
                case RDMA_INTR:
                        /*
                         * Failed because of an signal. Very likely
                         * the caller will not retry.
                         */
                        p->cku_err.re_status = RPC_INTR;
                        p->cku_err.re_errno = EINTR;
                        break;
                default:
                        /*
                         * All other failures - server down or service
                         * down or temporary resource failure. Delay before
                         * returning to caller.
                         */
                        ticks = clnt_rdma_min_delay * drv_usectohz(1000000);
                        p->cku_err.re_status = RPC_CANTCONNECT;
                        p->cku_err.re_errno = EIO;

                        if (h->cl_nosignal == TRUE) {
                                delay(ticks);
                        } else {
                                if (delay_sig(ticks) == EINTR) {
                                        p->cku_err.re_status = RPC_INTR;
                                        p->cku_err.re_errno = EINTR;
                                }
                        }
                        break;
                }

                return (p->cku_err.re_status);
        }

        if (p->cku_srcaddr.maxlen < conn->c_laddr.len) {
                if ((p->cku_srcaddr.maxlen != 0) &&
                    (p->cku_srcaddr.buf != NULL))
                        kmem_free(p->cku_srcaddr.buf, p->cku_srcaddr.maxlen);
                p->cku_srcaddr.buf = kmem_zalloc(conn->c_laddr.maxlen,
                    KM_SLEEP);
                p->cku_srcaddr.maxlen = conn->c_laddr.maxlen;
        }

        p->cku_srcaddr.len = conn->c_laddr.len;
        bcopy(conn->c_laddr.buf, p->cku_srcaddr.buf, conn->c_laddr.len);

        clnt_check_credit(conn);

        status = CLNT_RDMA_FAIL;

        rpcsec_gss = gss_i_or_p = FALSE;

        if (IS_RPCSEC_GSS(h)) {
                rpcsec_gss = TRUE;
                if (rpc_gss_get_service_type(h->cl_auth) ==
                    rpc_gss_svc_integrity ||
                    rpc_gss_get_service_type(h->cl_auth) ==
                    rpc_gss_svc_privacy)
                        gss_i_or_p = TRUE;
        }

        /*
         * Try a regular RDMA message if RPCSEC_GSS is not being used
         * or if RPCSEC_GSS is being used for authentication only.
         */
        if (rpcsec_gss == FALSE ||
            (rpcsec_gss == TRUE && gss_i_or_p == FALSE)) {
                /*
                 * Grab a send buffer for the request.  Try to
                 * encode it to see if it fits. If not, then it
                 * needs to be sent in a chunk.
                 */
                rpcmsg.type = SEND_BUFFER;
                if (rdma_buf_alloc(conn, &rpcmsg)) {
                        DTRACE_PROBE(krpc__e__clntrdma__callit_nobufs);
                        goto done;
                }

                /* First try to encode into regular send buffer */
                op = RDMA_MSG;

                call_xdrp = &callxdr;

                xdrrdma_create(call_xdrp, rpcmsg.addr, rpcmsg.len,
                    rdma_minchunk, NULL, XDR_ENCODE, conn);

                status = clnt_compose_rpcmsg(h, procnum, &rpcmsg, call_xdrp,
                    xdr_args, argsp);

                if (status != CLNT_RDMA_SUCCESS) {
                        /* Clean up from previous encode attempt */
                        rdma_buf_free(conn, &rpcmsg);
                        XDR_DESTROY(call_xdrp);
                } else {
                        XDR_CONTROL(call_xdrp, XDR_RDMA_GET_CHUNK_LEN, &rcil);
                }
        }

        /* If the encode didn't work, then try a NOMSG */
        if (status != CLNT_RDMA_SUCCESS) {

                msglen = CKU_HDRSIZE + BYTES_PER_XDR_UNIT + MAX_AUTH_BYTES +
                    xdr_sizeof(xdr_args, argsp);

                msglen = calc_length(msglen);

                /* pick up the lengths for the reply buffer needed */
                (void) xdrrdma_sizeof(xdr_args, argsp, 0,
                    &rcil.rcil_len, &rcil.rcil_len_alt);

                /*
                 * Construct a clist to describe the CHUNK_BUFFER
                 * for the rpcmsg.
                 */
                cl_rpcmsg = clist_alloc();
                cl_rpcmsg->c_len = msglen;
                cl_rpcmsg->rb_longbuf.type = RDMA_LONG_BUFFER;
                cl_rpcmsg->rb_longbuf.len = msglen;
                if (rdma_buf_alloc(conn, &cl_rpcmsg->rb_longbuf)) {
                        clist_free(cl_rpcmsg);
                        goto done;
                }
                cl_rpcmsg->w.c_saddr3 = cl_rpcmsg->rb_longbuf.addr;

                op = RDMA_NOMSG;
                call_xdrp = &callxdr;

                xdrrdma_create(call_xdrp, cl_rpcmsg->rb_longbuf.addr,
                    cl_rpcmsg->rb_longbuf.len, 0,
                    cl_rpcmsg, XDR_ENCODE, conn);

                status = clnt_compose_rpcmsg(h, procnum, &cl_rpcmsg->rb_longbuf,
                    call_xdrp, xdr_args, argsp);

                DTRACE_PROBE2(krpc__i__clntrdma__callit__longbuf, int, status,
                    int, msglen);
                if (status != CLNT_RDMA_SUCCESS) {
                        p->cku_err.re_status = RPC_CANTENCODEARGS;
                        p->cku_err.re_errno = EIO;
                        DTRACE_PROBE(krpc__e__clntrdma__callit__composemsg);
                        goto done;
                }
        }

        /*
         * During the XDR_ENCODE we may have "allocated" an RDMA READ or
         * RDMA WRITE clist.
         *
         * First pull the RDMA READ chunk list from the XDR private
         * area to keep it handy.
         */
        XDR_CONTROL(call_xdrp, XDR_RDMA_GET_RLIST, &cl);

        if (gss_i_or_p) {
                long_reply_len = rcil.rcil_len + rcil.rcil_len_alt;
                long_reply_len += MAX_AUTH_BYTES;
        } else {
                long_reply_len = rcil.rcil_len;
        }

        /*
         * Update the chunk size information for the Long RPC msg.
         */
        if (cl && op == RDMA_NOMSG)
                cl->c_len = p->cku_outsz;

        /*
         * Prepare the RDMA header. On success xdrs will hold the result
         * of xdrmem_create() for a SEND_BUFFER.
         */
        status = clnt_compose_rdma_header(conn, h, &clmsg,
            &rdmahdr_o_xdrs, &op);

        if (status != CLNT_RDMA_SUCCESS) {
                p->cku_err.re_status = RPC_CANTSEND;
                p->cku_err.re_errno = EIO;
                RCSTAT_INCR(rcnomem);
                DTRACE_PROBE(krpc__e__clntrdma__callit__nobufs2);
                goto done;
        }

        /*
         * Now insert the RDMA READ list iff present
         */
        status = clnt_setup_rlist(conn, rdmahdr_o_xdrs, call_xdrp);
        if (status != CLNT_RDMA_SUCCESS) {
                DTRACE_PROBE(krpc__e__clntrdma__callit__clistreg);
                rdma_buf_free(conn, &clmsg);
                p->cku_err.re_status = RPC_CANTSEND;
                p->cku_err.re_errno = EIO;
                goto done;
        }

        /*
         * Setup RDMA WRITE chunk list for nfs read operation
         * other operations will have a NULL which will result
         * as a NULL list in the XDR stream.
         */
        status = clnt_setup_wlist(conn, rdmahdr_o_xdrs, call_xdrp, &rndup);
        if (status != CLNT_RDMA_SUCCESS) {
                rdma_buf_free(conn, &clmsg);
                p->cku_err.re_status = RPC_CANTSEND;
                p->cku_err.re_errno = EIO;
                goto done;
        }

        /*
         * If NULL call and RPCSEC_GSS, provide a chunk such that
         * large responses can flow back to the client.
         * If RPCSEC_GSS with integrity or privacy is in use, get chunk.
         */
        if ((procnum == 0 && rpcsec_gss == TRUE) ||
            (rpcsec_gss == TRUE && gss_i_or_p == TRUE))
                long_reply_len += 1024;

        status = clnt_setup_long_reply(conn, &cl_long_reply, long_reply_len);

        DTRACE_PROBE2(krpc__i__clntrdma__callit__longreply, int, status,
            int, long_reply_len);

        if (status != CLNT_RDMA_SUCCESS) {
                rdma_buf_free(conn, &clmsg);
                p->cku_err.re_status = RPC_CANTSEND;
                p->cku_err.re_errno = EIO;
                goto done;
        }

        /*
         * XDR encode the RDMA_REPLY write chunk
         */
        seg_array_len = (cl_long_reply ? 1 : 0);
        (void) xdr_encode_reply_wchunk(rdmahdr_o_xdrs, cl_long_reply,
            seg_array_len);

        /*
         * Construct a clist in "sendlist" that represents what we
         * will push over the wire.
         *
         * Start with the RDMA header and clist (if any)
         */
        clist_add(&cl_sendlist, 0, XDR_GETPOS(rdmahdr_o_xdrs), &clmsg.handle,
            clmsg.addr, NULL, NULL);

        /*
         * Put the RPC call message in  sendlist if small RPC
         */
        if (op == RDMA_MSG) {
                clist_add(&cl_sendlist, 0, p->cku_outsz, &rpcmsg.handle,
                    rpcmsg.addr, NULL, NULL);
        } else {
                /* Long RPC already in chunk list */
                RCSTAT_INCR(rclongrpcs);
        }

        /*
         * Set up a reply buffer ready for the reply
         */
        status = rdma_clnt_postrecv(conn, p->cku_xid);
        if (status != RDMA_SUCCESS) {
                rdma_buf_free(conn, &clmsg);
                p->cku_err.re_status = RPC_CANTSEND;
                p->cku_err.re_errno = EIO;
                goto done;
        }

        /*
         * sync the memory for dma
         */
        if (cl != NULL) {
                status = clist_syncmem(conn, cl, CLIST_REG_SOURCE);
                if (status != RDMA_SUCCESS) {
                        (void) rdma_clnt_postrecv_remove(conn, p->cku_xid);
                        rdma_buf_free(conn, &clmsg);
                        p->cku_err.re_status = RPC_CANTSEND;
                        p->cku_err.re_errno = EIO;
                        goto done;
                }
        }

        /*
         * Send the RDMA Header and RPC call message to the server
         */
        status = RDMA_SEND(conn, cl_sendlist, p->cku_xid);
        if (status != RDMA_SUCCESS) {
                (void) rdma_clnt_postrecv_remove(conn, p->cku_xid);
                p->cku_err.re_status = RPC_CANTSEND;
                p->cku_err.re_errno = EIO;
                goto done;
        }

        /*
         * RDMA plugin now owns the send msg buffers.
         * Clear them out and don't free them.
         */
        clmsg.addr = NULL;
        if (rpcmsg.type == SEND_BUFFER)
                rpcmsg.addr = NULL;

        /*
         * Recv rpc reply
         */
        status = RDMA_RECV(conn, &cl_recvlist, p->cku_xid);

        /*
         * Now check recv status
         */
        if (status != 0) {
                if (status == RDMA_INTR) {
                        p->cku_err.re_status = RPC_INTR;
                        p->cku_err.re_errno = EINTR;
                        RCSTAT_INCR(rcintrs);
                } else if (status == RPC_TIMEDOUT) {
                        p->cku_err.re_status = RPC_TIMEDOUT;
                        p->cku_err.re_errno = ETIMEDOUT;
                        RCSTAT_INCR(rctimeouts);
                } else {
                        p->cku_err.re_status = RPC_CANTRECV;
                        p->cku_err.re_errno = EIO;
                }
                goto done;
        }

        /*
         * Process the reply message.
         *
         * First the chunk list (if any)
         */
        rdmahdr_i_xdrs = &(p->cku_inxdr);
        xdrmem_create(rdmahdr_i_xdrs,
            (caddr_t)(uintptr_t)cl_recvlist->w.c_saddr3,
            cl_recvlist->c_len, XDR_DECODE);

        /*
         * Treat xid as opaque (xid is the first entity
         * in the rpc rdma message).
         * Skip xid and set the xdr position accordingly.
         */
        XDR_SETPOS(rdmahdr_i_xdrs, sizeof (uint32_t));
        (void) xdr_u_int(rdmahdr_i_xdrs, &vers);
        (void) xdr_u_int(rdmahdr_i_xdrs, &rdma_credit);
        (void) xdr_u_int(rdmahdr_i_xdrs, &op);
        (void) xdr_do_clist(rdmahdr_i_xdrs, &cl);

        clnt_update_credit(conn, rdma_credit);

        wlist_exists_reply = FALSE;
        if (! xdr_decode_wlist(rdmahdr_i_xdrs, &cl_rpcreply_wlist,
            &wlist_exists_reply)) {
                DTRACE_PROBE(krpc__e__clntrdma__callit__wlist_decode);
                p->cku_err.re_status = RPC_CANTDECODERES;
                p->cku_err.re_errno = EIO;
                goto done;
        }

        /*
         * The server shouldn't have sent a RDMA_SEND that
         * the client needs to RDMA_WRITE a reply back to
         * the server.  So silently ignoring what the
         * server returns in the rdma_reply section of the
         * header.
         */
        (void) xdr_decode_reply_wchunk(rdmahdr_i_xdrs, &cl_rdma_reply);
        off = xdr_getpos(rdmahdr_i_xdrs);

        clnt_decode_long_reply(conn, cl_long_reply,
            cl_rdma_reply, &replyxdr, &reply_xdrp,
            cl, cl_recvlist, op, off);

        if (reply_xdrp == NULL)
                goto done;

        if (wlist_exists_reply) {
                XDR_CONTROL(reply_xdrp, XDR_RDMA_SET_WLIST, cl_rpcreply_wlist);
        }

        reply_msg.rm_direction = REPLY;
        reply_msg.rm_reply.rp_stat = MSG_ACCEPTED;
        reply_msg.acpted_rply.ar_stat = SUCCESS;
        reply_msg.acpted_rply.ar_verf = _null_auth;

        /*
         *  xdr_results will be done in AUTH_UNWRAP.
         */
        reply_msg.acpted_rply.ar_results.where = NULL;
        reply_msg.acpted_rply.ar_results.proc = xdr_void;

        /*
         * Decode and validate the response.
         */
        if (xdr_replymsg(reply_xdrp, &reply_msg)) {
                enum clnt_stat re_status;

                _seterr_reply(&reply_msg, &(p->cku_err));

                re_status = p->cku_err.re_status;
                if (re_status == RPC_SUCCESS) {
                        /*
                         * Reply is good, check auth.
                         */
                        if (!AUTH_VALIDATE(h->cl_auth,
                            &reply_msg.acpted_rply.ar_verf)) {
                                p->cku_err.re_status = RPC_AUTHERROR;
                                p->cku_err.re_why = AUTH_INVALIDRESP;
                                RCSTAT_INCR(rcbadverfs);
                                DTRACE_PROBE(
                                    krpc__e__clntrdma__callit__authvalidate);
                        } else if (!AUTH_UNWRAP(h->cl_auth, reply_xdrp,
                            xdr_results, resultsp)) {
                                p->cku_err.re_status = RPC_CANTDECODERES;
                                p->cku_err.re_errno = EIO;
                                DTRACE_PROBE(
                                    krpc__e__clntrdma__callit__authunwrap);
                        }
                } else {
                        /* set errno in case we can't recover */
                        if (re_status != RPC_VERSMISMATCH &&
                            re_status != RPC_AUTHERROR &&
                            re_status != RPC_PROGVERSMISMATCH)
                                p->cku_err.re_errno = EIO;

                        if (re_status == RPC_AUTHERROR) {
                                if ((refresh_attempt > 0) &&
                                    AUTH_REFRESH(h->cl_auth, &reply_msg,
                                    p->cku_cred)) {
                                        refresh_attempt--;
                                        try_call_again = 1;
                                        goto done;
                                }

                                try_call_again = 0;

                                /*
                                 * We have used the client handle to
                                 * do an AUTH_REFRESH and the RPC status may
                                 * be set to RPC_SUCCESS; Let's make sure to
                                 * set it to RPC_AUTHERROR.
                                 */
                                p->cku_err.re_status = RPC_AUTHERROR;

                                /*
                                 * Map recoverable and unrecoverable
                                 * authentication errors to appropriate
                                 * errno
                                 */
                                switch (p->cku_err.re_why) {
                                case AUTH_BADCRED:
                                case AUTH_BADVERF:
                                case AUTH_INVALIDRESP:
                                case AUTH_TOOWEAK:
                                case AUTH_FAILED:
                                case RPCSEC_GSS_NOCRED:
                                case RPCSEC_GSS_FAILED:
                                        p->cku_err.re_errno = EACCES;
                                        break;
                                case AUTH_REJECTEDCRED:
                                case AUTH_REJECTEDVERF:
                                default:
                                        p->cku_err.re_errno = EIO;
                                        break;
                                }
                        }
                        DTRACE_PROBE1(krpc__e__clntrdma__callit__rpcfailed,
                            int, p->cku_err.re_why);
                }
        } else {
                p->cku_err.re_status = RPC_CANTDECODERES;
                p->cku_err.re_errno = EIO;
                DTRACE_PROBE(krpc__e__clntrdma__callit__replymsg);
        }

done:
        clnt_return_credit(conn);

        if (cl_sendlist != NULL)
                clist_free(cl_sendlist);

        /*
         * If rpc reply is in a chunk, free it now.
         */
        if (cl_long_reply) {
                (void) clist_deregister(conn, cl_long_reply);
                rdma_buf_free(conn, &cl_long_reply->rb_longbuf);
                clist_free(cl_long_reply);
        }

        if (call_xdrp)
                XDR_DESTROY(call_xdrp);

        if (rndup.rb_private) {
                rdma_buf_free(conn, &rndup);
        }

        if (reply_xdrp) {
                (void) xdr_rpc_free_verifier(reply_xdrp, &reply_msg);
                XDR_DESTROY(reply_xdrp);
        }

        if (cl_rdma_reply) {
                clist_free(cl_rdma_reply);
        }

        if (cl_recvlist) {
                rdma_buf_t      recvmsg = {0};
                recvmsg.addr = (caddr_t)(uintptr_t)cl_recvlist->w.c_saddr3;
                recvmsg.type = RECV_BUFFER;
                RDMA_BUF_FREE(conn, &recvmsg);
                clist_free(cl_recvlist);
        }

        RDMA_REL_CONN(conn);

        if (try_call_again)
                goto call_again;

        if (p->cku_err.re_status != RPC_SUCCESS) {
                RCSTAT_INCR(rcbadcalls);
        }
        return (p->cku_err.re_status);
}


static void
clnt_decode_long_reply(CONN *conn,
    struct clist *cl_long_reply,
    struct clist *cl_rdma_reply, XDR *xdrs,
    XDR **rxdrp, struct clist *cl,
    struct clist *cl_recvlist,
    uint_t  op, uint_t off)
{
        if (op != RDMA_NOMSG) {
                DTRACE_PROBE1(krpc__i__longrepl__rdmamsg__len,
                    int, cl_recvlist->c_len - off);
                xdrrdma_create(xdrs,
                    (caddr_t)(uintptr_t)(cl_recvlist->w.c_saddr3 + off),
                    cl_recvlist->c_len - off, 0, cl, XDR_DECODE, conn);
                *rxdrp = xdrs;
                return;
        }

        /* op must be RDMA_NOMSG */
        if (cl) {
                DTRACE_PROBE(krpc__e__clntrdma__declongreply__serverreadlist);
                return;
        }

        if (cl_long_reply->u.c_daddr) {
                DTRACE_PROBE1(krpc__i__longrepl__rdmanomsg__len,
                    int, cl_rdma_reply->c_len);

                xdrrdma_create(xdrs, (caddr_t)cl_long_reply->u.c_daddr3,
                    cl_rdma_reply->c_len, 0, NULL, XDR_DECODE, conn);

                *rxdrp = xdrs;
        }
}

static void
clnt_return_credit(CONN *conn)
{
        rdma_clnt_cred_ctrl_t *cc_info = &conn->rdma_conn_cred_ctrl_u.c_clnt_cc;

        mutex_enter(&conn->c_lock);
        cc_info->clnt_cc_in_flight_ops--;
        cv_signal(&cc_info->clnt_cc_cv);
        mutex_exit(&conn->c_lock);
}

static void
clnt_update_credit(CONN *conn, uint32_t rdma_credit)
{
        rdma_clnt_cred_ctrl_t *cc_info = &conn->rdma_conn_cred_ctrl_u.c_clnt_cc;

        /*
         * If the granted has not altered, avoid taking the
         * mutex, to essentially do nothing..
         */
        if (cc_info->clnt_cc_granted_ops == rdma_credit)
                return;
        /*
         * Get the granted number of buffers for credit control.
         */
        mutex_enter(&conn->c_lock);
        cc_info->clnt_cc_granted_ops = rdma_credit;
        mutex_exit(&conn->c_lock);
}

static void
clnt_check_credit(CONN *conn)
{
        rdma_clnt_cred_ctrl_t *cc_info = &conn->rdma_conn_cred_ctrl_u.c_clnt_cc;

        /*
         * Make sure we are not going over our allowed buffer use
         * (and make sure we have gotten a granted value before).
         */
        mutex_enter(&conn->c_lock);
        while (cc_info->clnt_cc_in_flight_ops >= cc_info->clnt_cc_granted_ops &&
            cc_info->clnt_cc_granted_ops != 0) {
                /*
                 * Client has maxed out its granted buffers due to
                 * credit control.  Current handling is to block and wait.
                 */
                cv_wait(&cc_info->clnt_cc_cv, &conn->c_lock);
        }
        cc_info->clnt_cc_in_flight_ops++;
        mutex_exit(&conn->c_lock);
}

/* ARGSUSED */
static void
clnt_rdma_kabort(CLIENT *h)
{
}

static void
clnt_rdma_kerror(CLIENT *h, struct rpc_err *err)
{
        struct cku_private *p = htop(h);
        *err = p->cku_err;
}

static bool_t
clnt_rdma_kfreeres(CLIENT *h, xdrproc_t xdr_res, caddr_t res_ptr)
{
        struct cku_private *p = htop(h);
        XDR *xdrs;

        xdrs = &(p->cku_outxdr);
        xdrs->x_op = XDR_FREE;
        return ((*xdr_res)(xdrs, res_ptr));
}

/* ARGSUSED */
static bool_t
clnt_rdma_kcontrol(CLIENT *h, int cmd, char *arg)
{
        return (TRUE);
}

/* ARGSUSED */
static int
clnt_rdma_ksettimers(CLIENT *h, struct rpc_timers *t, struct rpc_timers *all,
        int minimum, void(*feedback)(int, int, caddr_t), caddr_t arg,
        uint32_t xid)
{
        RCSTAT_INCR(rctimers);
        return (0);
}

int
rdma_reachable(int addr_type, struct netbuf *addr, struct knetconfig **knconf)
{
        rdma_registry_t *rp;
        void *handle = NULL;
        struct knetconfig *knc;
        char *pf, *p;
        rdma_stat status;
        int error = 0;

        if (!INGLOBALZONE(curproc))
                return (-1);

        /*
         * modload the RDMA plugins if not already done.
         */
        if (!rdma_modloaded) {
                mutex_enter(&rdma_modload_lock);
                if (!rdma_modloaded) {
                        error = rdma_modload();
                }
                mutex_exit(&rdma_modload_lock);
                if (error)
                        return (-1);
        }

        if (!rdma_dev_available)
                return (-1);

        rw_enter(&rdma_lock, RW_READER);
        rp = rdma_mod_head;
        while (rp != NULL) {
                if (rp->r_mod_state == RDMA_MOD_INACTIVE) {
                        rp = rp->r_next;
                        continue;
                }
                status = RDMA_REACHABLE(rp->r_mod->rdma_ops, addr_type, addr,
                    &handle);
                if (status == RDMA_SUCCESS) {
                        knc = kmem_zalloc(sizeof (struct knetconfig),
                            KM_SLEEP);
                        knc->knc_semantics = NC_TPI_RDMA;
                        pf = kmem_alloc(KNC_STRSIZE, KM_SLEEP);
                        p = kmem_alloc(KNC_STRSIZE, KM_SLEEP);
                        if (addr_type == AF_INET)
                                (void) strncpy(pf, NC_INET, KNC_STRSIZE);
                        else if (addr_type == AF_INET6)
                                (void) strncpy(pf, NC_INET6, KNC_STRSIZE);
                        pf[KNC_STRSIZE - 1] = '\0';

                        (void) strncpy(p, rp->r_mod->rdma_api, KNC_STRSIZE);
                        p[KNC_STRSIZE - 1] = '\0';

                        knc->knc_protofmly = pf;
                        knc->knc_proto = p;
                        knc->knc_rdev = (dev_t)rp;
                        *knconf = knc;
                        rw_exit(&rdma_lock);
                        return (0);
                }
                rp = rp->r_next;
        }
        rw_exit(&rdma_lock);
        return (-1);
}